Support for HAPROXY load balancer (#6)

* 1) Added support for HAPROXY load balancer and example in docker-compose
2) Fixed Readme

* 1) Fixed compose and README.md for it.

* Forgot to fix in click

Co-authored-by: Yatskov <[email protected]>

README.md

@@ -21,6 +21,10 @@ So you should use `docker run --cap-add=SYS_ADMIN` option.
 $ docker run -d -p 3000:3000 --name scrapy-puppeter-service --cap-add SYS_ADMIN isprascrawlers/scrapy-puppeteer-service 
 ```
 
+To run example which shows how to deploy several instances of service with load balancer use this command.
+```shell script
+$ docker-compose up -d
+```
 
 ## API
 
@@ -29,10 +33,9 @@ For All requests puppeteer browser creates new incognito browser context and new
 If your want to reuse your browser context simple send context_id in your query. 
 All request return their context ids in response. 
 Also you could reuse your browser page and more actions with it.
-In order to so you should send closePage=false in your previous request, that would made service save current page and 
-return its page_id. 
-If you want to send request to this use page_id and context_id params in your next request.
-
+In order to do so you should send in your request pageId that is returned in your previous request,
+that would make service reuse current page and return again its pageId. 
+If you want to close the page you are working with you should send in query param "closePage" with non-empty value.
 If you want your requests on page make through proxy, just add to normal request "proxy" param. 
 Proxy username and password params are optional.
 Also you can add extra http headers to each request that is made on page.
@@ -153,4 +156,4 @@ You need to explicitly close the browser tab once you don't need it (e.g. at the
 - [x] support of extra headers
 - [ ] error handling for requests
 - [ ] har support
-- [ ] scaling to several docker containers
+- [x] scaling to several docker containers

docker-compose.dev.yml

@@ -0,0 +1,15 @@
+version: "3.7"
+
+services:
+
+  scrapy-puppeteer-service-1:
+    build:
+      context: .
+    ports:
+      - "3001:3000"
+
+  scrapy-puppeteer-service-2:
+    build:
+      context: .
+    ports:
+      - "3002:3000"

docker-compose.yml

@@ -0,0 +1,25 @@
+version: "3.7"
+
+services:
+  haproxy:
+    image: "haproxy"
+    ports:
+      - "3000:3000"
+    volumes:
+      - "./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"
+
+  scrapy-puppeteer-service-1:
+    image: "isprascrawlers/scrapy-puppeteer-service"
+    environment:
+      TIMEOUT: 600000
+    cap_add:
+      - SYS_ADMIN
+    restart: always
+
+  scrapy-puppeteer-service-2:
+    image: "isprascrawlers/scrapy-puppeteer-service"
+    environment:
+      TIMEOUT: 600000
+    cap_add:
+      - SYS_ADMIN
+    restart: always

haproxy/haproxy.cfg

@@ -0,0 +1,15 @@
+frontend entrypoint
+bind 0.0.0.0:3000
+mode http
+default_backend balancer
+
+backend balancer
+mode http
+option httpclose
+balance leastconn
+stick-table type string len 64 size 100k expire 15m
+stick store-response res.hdr(scrapy-puppeteer-service-context-id)
+stick match urlp(contextId)
+server app1 scrapy-puppeteer-service-1:3000 check
+server app2 scrapy-puppeteer-service-2:3000 check
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrapy-puppeteer-service",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "private": true,
   "scripts": {
     "start": "node ./bin/www"

routes/action.js

@@ -37,8 +37,8 @@ router.post('/', async function (req, res, next) {
         }
 
         let response = await utils.perfomAction(req, action);
-        res.setHeader('Content-Type', 'application/json');
-        res.send(JSON.stringify(response));
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
+        res.send(response);
     } catch (e) {
         next(e);
     }

routes/click.js

@@ -44,8 +44,8 @@ router.post('/', async function (req, res, next) {
 
     try {
         let response = await utils.perfomAction(req, action);
-        res.setHeader('Content-Type', 'application/json');
-        res.send(JSON.stringify(response));
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
+        res.send(response)
     } catch (e) {
         next(e);
     }

routes/goback.js

@@ -19,9 +19,8 @@ router.post('/', async function (req, res, next) {
 
     try {
         let response = await utils.perfomAction(req, action);
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response)
-        // res.setHeader('Content-Type', 'application/json');
-        // res.send(JSON.stringify(response));
     } catch (e) {
         next(e);
     }

routes/goforward.js

@@ -19,9 +19,8 @@ router.post('/', async function (req, res, next) {
 
     try {
         let response = await utils.perfomAction(req, action);
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response)
-        // res.setHeader('Content-Type', 'application/json');
-        // res.send(JSON.stringify(response));
     } catch (e) {
         next(e);
     }

routes/goto.js

@@ -32,6 +32,7 @@ router.post('/', async function (req, res, next) {
 
     try {
         let response = await utils.perfomAction(req, action);
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response);
     } catch (e) {
         next(e);

routes/har.js

@@ -15,6 +15,7 @@ router.post('/', async function (req, res, next) {
 
     try {
         let response = await utils.perfomAction(req, action);
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response);
     } catch (e) {
         next(e);

routes/screenshot.js

@@ -19,8 +19,8 @@ router.post('/', async function (req, res, next) {
 
     try {
         let response = await utils.perfomAction(req, action);
-        res.setHeader('Content-Type', 'application/json');
-        res.send(JSON.stringify(response));
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
+        res.send(response);
     } catch (e) {
         next(e);
     }

routes/scroll.js

@@ -30,8 +30,8 @@ router.post('/', async function (req, res, next) {
 
     try {
         let response = await utils.perfomAction(req, action);
-        res.setHeader('Content-Type', 'application/json');
-        res.send(JSON.stringify(response));
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
+        res.send(response);
     } catch (e) {
         next(e);
     }