Add partitioning logic
This commit is contained in:
parent
5f773d46c9
commit
77465e6b49
6 changed files with 168 additions and 0 deletions
23
src/partition/partitionconfig.js
Normal file
23
src/partition/partitionconfig.js
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
class PartitionConfig {
|
||||
constructor(config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
getPartitionMap() {
|
||||
return this.config.partitions;
|
||||
}
|
||||
|
||||
getOverrideMap() {
|
||||
return this.config.overrides;
|
||||
}
|
||||
|
||||
getPool() {
|
||||
return this.config.pool;
|
||||
}
|
||||
|
||||
getIdentity() {
|
||||
return this.config.identity;
|
||||
}
|
||||
}
|
||||
|
||||
export { PartitionConfig };
|
||||
29
src/partition/partitiondecider.js
Normal file
29
src/partition/partitiondecider.js
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import { murmurHash1 } from '../util/murmur';
|
||||
|
||||
class PartitionDecider {
|
||||
constructor(config) {
|
||||
this.identity = config.getIdentity();
|
||||
this.partitionMap = config.getPartitionMap();
|
||||
this.pool = config.getPool();
|
||||
this.overrideMap = config.getOverrideMap();
|
||||
}
|
||||
|
||||
getPartitionForChannel(channel) {
|
||||
return this.partitionMap[this.getPartitionIdentityForChannel(channel)];
|
||||
}
|
||||
|
||||
getPartitionIdentityForChannel(channel) {
|
||||
if (this.overrideMap.hasOwnProperty(channel)) {
|
||||
return this.overrideMap[channel];
|
||||
} else {
|
||||
const i = murmurHash1(channel) % this.pool.length;
|
||||
return this.pool[i];
|
||||
}
|
||||
}
|
||||
|
||||
isChannelOnThisPartition(channel) {
|
||||
return this.getPartitionIdentityForChannel(channel) === this.identity;
|
||||
}
|
||||
}
|
||||
|
||||
export { PartitionDecider };
|
||||
63
src/partition/partitionmodule.js
Normal file
63
src/partition/partitionmodule.js
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import { loadFromToml } from 'cytube-common/lib/configuration/configloader';
|
||||
import { PartitionConfig } from './partitionconfig';
|
||||
import { PartitionDecider } from './partitiondecider';
|
||||
import { PartitionClusterClient } from '../io/cluster/partitionclusterclient';
|
||||
import logger from 'cytube-common/lib/logger';
|
||||
import LegacyConfig from '../config';
|
||||
import path from 'path';
|
||||
|
||||
const PARTITION_CONFIG_PATH = path.resolve(__dirname, '..', '..', 'conf',
|
||||
'partitions.toml');
|
||||
|
||||
class PartitionModule {
|
||||
constructor() {
|
||||
this.initConfig();
|
||||
}
|
||||
|
||||
onReady() {
|
||||
|
||||
}
|
||||
|
||||
initConfig() {
|
||||
logger.initialize(null, null, LegacyConfig.get('debug'));
|
||||
try {
|
||||
this.partitionConfig = this.loadPartitionMap();
|
||||
} catch (error) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
loadPartitionMap() {
|
||||
try {
|
||||
return loadFromToml(PartitionConfig, PARTITION_CONFIG_PATH);
|
||||
} catch (error) {
|
||||
if (typeof error.line !== 'undefined') {
|
||||
logger.error(`Error in ${PARTITION_CONFIG_PATH}: ${error} ` +
|
||||
`(line ${error.line})`);
|
||||
} else {
|
||||
logger.error(`Error loading ${PARTITION_CONFIG_PATH}: ` +
|
||||
`${error.stack}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
getPartitionDecider() {
|
||||
if (!this.partitionDecider) {
|
||||
this.partitionDecider = new PartitionDecider(this.partitionConfig);
|
||||
}
|
||||
|
||||
return this.partitionDecider;
|
||||
}
|
||||
|
||||
getClusterClient() {
|
||||
if (!this.partitionClusterClient) {
|
||||
this.partitionClusterClient = new PartitionClusterClient(
|
||||
this.getPartitionDecider());
|
||||
}
|
||||
|
||||
return this.partitionClusterClient;
|
||||
}
|
||||
}
|
||||
|
||||
export { PartitionModule };
|
||||
Loading…
Add table
Add a link
Reference in a new issue