-
Notifications
You must be signed in to change notification settings - Fork 0
/
server.js
168 lines (133 loc) · 8.52 KB
/
server.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
const express = require("express");
const app = express();
const scrap = require("./algorithm");
const event = scrap.defEvent;
const ms_connect = scrap.ms_connect;
const server = app.listen(8000, function(){ console.log('Listening on 8000'); });
const io = require("socket.io").listen(server);
const internetAvailable = require("internet-available");
app.use(express.static(__dirname + "/"));
app.get("/scrap",function(req,res){
res.sendFile(__dirname+"/index.html");//Set the Default Route
io.on("connection",function(socket){ //On Socket Connection
socketSameMess("Socket",'Sockets Connection Made on ID : <span style="color:#03a9f4;">'+socket.id+'<span>');
ms_connect.connect(function(err){//On Connection with Database
if(err) socketSameMess("database_error",err+" "); // If any error in database connection
socketSameMess("Database ",'Connected to MYSQL Database Successfully...');
/************************************************************************************************************************************
******************************************************** START OR INITIATE SCRAPPING*************************************************
************************************************************************************************************************************/
socket.on("initScrapHit",function(flag){
if(flag == "START_SCRAPPING"){
ms_connect.query('SELECT * FROM companies',function(err,rows,fields){
if(err) console.log("Error Executing Query");
socketSameMess("Database ",'Sql Query Executed, Data Recieved Successfully !');
socketSameMess("Database ",'Total Websites to Scrap : ' + rows.length);
var COMPANIES = rows,
COMPANY_DATA,
C = 0, //Rows array index
NEXT_SITE = false;
socket.emit("all_sites",rows);
var scrapWebsite = (function scrapWeb(){
internetAvailable().then(function(){
socketSameMess("Internet",'Established internet connection !');
if(NEXT_SITE){
NEXT_SITE = false;
C++;
scrapWebsite();
} else{
CODES_REF = COMPANIES[C].id;
ms_connect.query("SELECT * FROM companies_codes WHERE `companies_codes`.`comp_fk` ="+CODES_REF,function(err,result){
if(err) console.log(err);
event.emit("hitDefConsole",result);
COMPANY_DATA = result;
var s = new scrap.ScrapData({
info : COMPANIES[C],
data : COMPANY_DATA[0]
});
})
}
}).catch(function(){
socketSameMess("Internet",'There is no Internet Connection Available!');
})
return scrapWeb;
})();
event.on("siteScrapped",function(){
NEXT_SITE = true;
scrapWebsite();
});
});
}
else if(flag == "PAUSE_SCRAPPING"){
event.emit("pauseScrapping",true);
event.on("pauseSuccess",function(data){
socket.emit("pauseSuccess",data);
socketSameMess("Scrapper ","Paused Info : <span class='bold'>"+ data.time + "</span> and date : <span class='bold'>" +data.dateText + "</span> with total pages scrapped = <span class='bold'>" + data.page +"</span>");
// ms_connect.query("INSERT INTO scrapper_paused_data VALUES () ");
});
}
else if(flag == "STOP_SCRAPPING"){
event.emit("stopScrapping",true);
}
}); //initiates Scrapping
/************************************************************************************************************************************
********************************************************ADD NEW COMPANIES TO DATABASE************************************************
************************************************************************************************************************************/
socket.on("initCompanyAdd",function(data){
var ac = new scrap.AddCompany(data);
event.on("site-add-error",function(data){
socket.emit("site-add-error",data);
});
}); // adding a new company
/************************************************************************************************************************************
********************************************************GET ALL STATISTICS OF THE SITE***********************************************
************************************************************************************************************************************/
event.on("initCompStats",function(data){
var ss = new scrap.ScrapperStats();
event.emit("hitDefConsole",ss.currentStats());
}); // Retrieving Companies Statistics
/************************************************************************************************************************************
********************************************************SAVE SCRAPER SETTINGS********************************************************
************************************************************************************************************************************/
socket.on("initScrapSettings",function(data){ scrapperSettings(data); }); //Initializing Scrapper Settings
/************************************************************************************************************************************
********************************************************GET ALL STATISTICS OF THE SITE***********************************************
************************************************************************************************************************************/
socket.on("getCompanies", function(){
ms_connect.query('SELECT * FROM companies',function(err,data,f){
if(err) throw err;
socket.emit("parseCompanies",data);
})
}); // Retrieving Companies
})
event.on("hitConsole",function(data){
socketSameMess(data.a,data.m);
});
event.on("hitBar",function(data){
socket.emit("hitBar",data);
});
event.on("hitDefConsole",function(data){
socket.emit("hitDefConsole",data);
})
event.on("UIMetaUpdates",function(data){
socket.emit("UIMetaUpdates",data);
});
event.on("getTotalTime",function(){
socket.emit("getTotalTime");
socket.on("showTotalTime",function(data){
event.emit("showTotalTime",data);
})
});
function socketSameMess(auth,mess){
socket.emit("consoleData",{a:auth,m:mess});
}
io.on("disconnect",function(){
console.log("Client Disconnected !");
scrap.ms_connect.end();
socket.removeAllListeners();
})
})
})
// res.setHeader('Access-Control-Allow-Origin', '*');
// res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE');
// res.setHeader('Access-Control-Allow-Headers', 'X-Requested-With,content-type');