docker-compose fixed

data/config.yaml

@@ -20,12 +20,12 @@ services:
     service: RQExecutor
     storage: pool
     redis:
-      host: "127.0.0.1"
+      host: redis
   tg_feed:
     package: lib.plugin.base.lib.Telegram
     service: TelegramFeed
     storage: pool
-    token: "358947212354:"
+    token: "3"
     chats:
     - id: aiWeipeighah7vufoHa0ieToipooYe
       if:
@@ -40,12 +40,14 @@ storage:
   pool:
     package: lib.plugin.base.lib.Mongo
     service: MongoStorage
+    url: mongo
     size: 0
     db: "medved"
     coll: 'pool'
   ip_source:
     package: lib.plugin.base.lib.Mongo
     service: MongoStorage
+    url: mongo
     size: 800
     db: "medved"
     coll: 'ip_source'
@@ -150,5 +152,5 @@ tasks:
     path: lib/plugin/iscan/templates/ftp.tpl
 
 logging: 
-  Storage: DEBUG
+  Storage: INFO
-  Loader: DEBUG
+  Loader: INFO

docker-compose.yml

@@ -51,6 +51,8 @@ services:
 
   mongo:
     image: mongo:latest
+    command:
+      - '--quiet'
     volumes:
       - ./docker/lib/mongo:/data/
     env_file:

docker/core/files/run.sh

@@ -7,6 +7,7 @@ export REDIS_IP=$(host ${REDIS_IP} | head -n1 | grep -Po "(\d+\.?){4}")
 
 /tmp/confd -onetime -backend env
 
-sudo -u tor tor
+#sudo -u tor tor
 
-cd /mdvd && proxychains -q python3 medved.py
+#cd /mdvd && proxychains -q python3 medved.py
+cd /mdvd && python3 medved.py

docker/worker/Dockerfile

@@ -1,5 +1,16 @@
 FROM medved_base:latest
 
+RUN pacman -S --noconfirm --needed git libpcap linux-headers clang tor
+
+RUN git clone https://github.com/robertdavidgraham/masscan && \
+    cd masscan && \
+    make -j && \
+    mv bin/masscan /usr/bin/masscan
+
+RUN wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz && gunzip GeoLiteCity.dat.gz
+
+RUN mkdir -p /usr/share/GeoIP/ && mv GeoLiteCity.dat /usr/share/GeoIP/GeoIPCity.dat
+
 ADD files/run.sh /tmp/run.sh
 
 CMD ["/tmp/run.sh"]

docker/worker/files/run.sh

@@ -4,4 +4,4 @@ export CORE_IP=$(host ${CORE_IP} | head -n1 | grep -Po "(\d+\.?){4}")
 
 /tmp/confd -onetime -backend env
 
-cd /mdvd && proxychains -q rq worker high normal low -u "redis://${REDIS_IP}:6379/"
+cd /mdvd && rq worker high normal low -u "redis://${REDIS_IP}:6379/"

lib/plugin/iscan/tasks/common.py

@@ -42,7 +42,7 @@ class MasScanTask(Task):
     
     self._logger.debug("Starting scan, ip_list=%s, port_list=%s", ip_list, port_list)
     
-    ms = MasScan()
+    ms = MasScan(bin_path=self.lcnf.get('bin_path', "/usr/bin/masscan"))
     hosts = ms.scan(ip_list=ip_list, port_list=port_list)
     
     self._logger.debug(hosts)

lib/plugin/iscan/tasks/ftp.py

@@ -94,7 +94,7 @@ class FTPListFilesTask(Task):
 
     try:
       if len(filelist) == 0 or filelist[0] == "total 0":
-        raise self.PipelineError("Empty server")
+        item['data']['filter'] = "Empty server"
     except IndexError:
       pass
 
@@ -122,4 +122,4 @@ class FTPListFilesTask(Task):
       self._process(item)
       if self.lcnf.get('filter', False):
         self._filter(item)
-    return items
+    return items

lib/plugin/iscan/tasks/gopher.py

@@ -23,6 +23,7 @@ class GopherFindTask(Task):
     return ''.join(total_data)
 
   def _process(self, item):
+    item['steps'][self._id] = False
     sock = socket.socket()
     sock.settimeout(self.lcnf.get('timeout', 20))
     sock.connect((item['data']['ip'], int(70)))
@@ -46,5 +47,5 @@ class GopherFindTask(Task):
         item['data']['files'].append(node)
 
     if not item['data']['files']:
-      item['steps'][self._id] = False
       raise Exception("Empty server (not Gopher?)")
+    item['steps'][self._id] = True

requirements.txt

@@ -11,4 +11,5 @@ urllib3
 zmq
 jsoncomment
 rq
-pyyaml
+pyyaml
+jinja2