Repository navigation
Expand file tree
/
Copy pathmanager.js
More file actions
100 lines (91 loc) · 2.96 KB
/
Copy pathmanager.js
File metadata and controls
100 lines (91 loc) · 2.96 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
const cluster = require('cluster');
var os = require('os');
const search = require('./index.js');
const moment = require('moment');
const deleteDups = require('./deleteDups.js');
const models = require('./MongoModels.js');
let memUsed = 125000000;
var max_processes = Math.floor(os.freemem()/memUsed);
let current_processes = 0;
let urls = [];
let initURLs = ["https://moz.com/top500", "https://gist.github.com/demersdesigns/4442cd84c1cc6c5ccda9b19eac1ba52b", "https://ahrefs.com/blog/most-visited-websites/", "https://fossbytes.com/most-useful-websites-internet/"];
require('console-stamp')(console, {
metadata: function() {
return ('[' + process.pid + ']');
},
colors: {
stamp: 'yellow',
label: 'white',
metadata: 'green'
}
});
function newProccess(url) {
if (current_processes < max_processes) {
cluster.fork({ URL: url });
urls.shift();
current_processes++;
}
}
cluster.on('fork', function(worker) {
console.log("Child process " + worker.process.pid + " created");
});
cluster.on('exit', async function(worker, code, signal) {
console.log("Child process " + worker.process.pid + " exited");
current_processes--;
if (urls.length === 0 || urls[0] === undefined || urls[0] === null) {
urls = (await models.Url.find().exec());
let random = Math.floor(Math.random() * urls.length);
urls = (await models.Url.find().skip(random).limit(20).exec());
console.log("Retrieved " + urls.length + " URLs from the database");
}
if (urls.length === 0 || urls[0] === undefined || urls[0] === null) {
for (let k = 0; k < initURLs.length; k++) {
urls.push({ Url: initURLs[k]})
}
}
newProccess(urls[0].Url);
});
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
if (cluster.isMaster) {
masterProcess();
} else {
childProcess();
}
async function masterProcess() {
dbconn = require('./databaseConnect.js');
console.log(`Manager is running`);
setInterval(function() {
}, 2500);
dbconn.connect();
await sleep(500);
urls = (await models.Url.find().exec());
let random = Math.floor(Math.random() * urls.length);
urls = (await models.Url.find().skip(random).limit(20).exec());
console.log("Retrieved " + urls.length + " URLs from the database");
if (urls.length <= 0) {
for (let k = 0; k < initURLs.length; k++) {
urls.push({ Url: initURLs[k]})
}
}
for (let i = 0; i < max_processes; i++) {
if (urls[0]) {
newProccess(urls[0].Url);
urls.shift();
}
}
}
async function childProcess() {
require('./databaseConnect.js').connect();
const outcome = await search(process.env.URL, process.pid);
if (outcome === search.OUTCOMES.DEFERRED) {
console.log("Kept URL: " + process.env.URL + " in queue (" + outcome + ")");
} else {
await models.Url.deleteMany({ Url: process.env.URL });
console.log("Removed URL: " + process.env.URL + " from queue (" + outcome + ")");
}
process.exit(0);
}