From d0ff66993d9688b5a5ff43909be32d6ea0d42c2f Mon Sep 17 00:00:00 2001 From: Good Evening Date: Thu, 2 Aug 2018 20:42:09 +0300 Subject: [PATCH] FTP pipeline works --- Config.py | 13 +++- data/config.yaml | 100 +++++++++++++++++++++++--- lib/data/Item.py | 17 +++++ lib/data/Storage.py | 15 ++++ lib/data/__init__.py | 5 +- lib/exeq/Executor.py | 41 ++++++----- lib/plugin/Manager.py | 9 --- lib/plugin/README | 6 -- lib/plugin/__init__.py | 1 - lib/plugin/base/lib/IP.py | 4 +- lib/plugin/base/lib/Mongo.py | 34 +++++---- lib/plugin/base/lib/Telegram.py | 29 +++----- lib/plugin/base/tasks/text.py | 17 +++++ lib/plugin/iscan/plugin.yaml | 62 ---------------- lib/plugin/iscan/tasks/common.py | 2 - lib/plugin/iscan/tasks/ftp.py | 4 ++ lib/plugin/iscan/tasks/gopher.py | 50 +++++++++++++ lib/plugin/iscan/templates/ftp.tpl | 6 ++ lib/plugin/iscan/templates/gopher.tpl | 11 +++ lib/plugin/plugins/FTP.py | 18 ----- lib/plugin/plugins/Gopher.py | 45 ------------ lib/plugin/plugins/README | 1 - lib/plugin/plugins/__init__.py | 0 lib/plugin/test.py | 45 ------------ lib/util.py | 6 +- 25 files changed, 281 insertions(+), 260 deletions(-) create mode 100644 lib/data/Item.py delete mode 100644 lib/plugin/Manager.py delete mode 100644 lib/plugin/README delete mode 100644 lib/plugin/__init__.py create mode 100644 lib/plugin/base/tasks/text.py delete mode 100644 lib/plugin/iscan/plugin.yaml create mode 100644 lib/plugin/iscan/tasks/gopher.py create mode 100644 lib/plugin/iscan/templates/ftp.tpl create mode 100644 lib/plugin/iscan/templates/gopher.tpl delete mode 100644 lib/plugin/plugins/FTP.py delete mode 100644 lib/plugin/plugins/README delete mode 100644 lib/plugin/plugins/__init__.py delete mode 100644 lib/plugin/test.py diff --git a/Config.py b/Config.py index 3008191..e26807d 100644 --- a/Config.py +++ b/Config.py @@ -3,6 +3,13 @@ import yaml -cnf = {} -with open('data/config.yaml') as config_file: - cnf = yaml.load(config_file) \ No newline at end of file +class Config(object): + def __init__(self): + with open('data/config.yaml') as config_file: + self.config = yaml.load(config_file) + + def get(self, key, defval=None): + return self.config.get(key, defval) + + +cnf = Config() \ No newline at end of file diff --git a/data/config.yaml b/data/config.yaml index c1a8d73..5453692 100644 --- a/data/config.yaml +++ b/data/config.yaml @@ -5,20 +5,32 @@ core: services: - random_ip - rq_executor + - tg_feed pipelines: - ftp + - gopher services: + random_ip: + package: lib.plugin.base.lib.IP + service: RandomIP + storage: ip_source rq_executor: package: lib.exeq.Executor service: RQExecutor storage: pool redis: host: "127.0.0.1" - random_ip: - package: lib.plugin.base.lib.IP - service: RandomIP - storage: ftp_source + tg_feed: + package: lib.plugin.base.lib.Telegram + service: TelegramFeed + storage: pool + token: "358947514:" + chats: + - id: aiWeipeighah7vufoHa0ieToipooYe + if: + steps.ftp_apply_tpl: true + data.filter: false storage: pool: @@ -27,21 +39,21 @@ storage: size: 0 db: "medved" coll: 'pool' - ftp_source: + ip_source: package: lib.plugin.base.lib.Mongo service: MongoStorage - size: 500 + size: 800 db: "medved" - coll: 'ftp_source' + coll: 'ip_source' pipelines: ftp: - source: ftp_source + source: ip_source steps: - task: ftp_scan priority: low - multiple: 100 + parallel: 100 - task: ftp_connect priority: normal if: @@ -50,14 +62,74 @@ pipelines: priority: high if: steps.ftp_connect: true + - task: ftp_apply_tpl + priority: high + if: + steps.ftp_list_files: true + gopher: + source: ip_source + steps: + - task: gopher_scan + priority: normal + parallel: 100 + - task: gopher_find + priority: high + if: + steps.gopher_scan: true + - task: gopher_apply_tpl + priority: high + if: + steps.gopher_find: true + + http: + source: ip_source + steps: + - task: http_scan + priority: low + parallel: 25 tasks: + gopher_scan: + package: lib.plugin.iscan.tasks.common + service: MasScanTask + ports: + - 70 + gopher_find: + package: lib.plugin.iscan.tasks.gopher + service: GopherFindTask + gopher_apply_tpl: + package: lib.plugin.base.tasks.text + service: Jinja2TemplateTask + path: lib/plugin/iscan/templates/gopher.tpl + + vnc_scan: + package: lib.plugin.iscan.tasks.common + service: MasScanTask + ports: + - 5900 + - 5901 + vnc_connect: + package: lib.plugin.iscan.tasks.vnc + service: VNCConnectTask + ports: + - 5900 + - 5901 + + http_scan: + package: lib.plugin.iscan.tasks.common + service: MasScanTask + ports: + - 80 + - 81 + - 8080 + - 8081 + ftp_scan: package: lib.plugin.iscan.tasks.common service: MasScanTask ports: - 21 - ftp_connect: + ftp_connect: package: lib.plugin.iscan.tasks.ftp service: FTPConnectTask logins: data/ftp/logins.txt @@ -67,6 +139,12 @@ tasks: ftp_list_files: package: lib.plugin.iscan.tasks.ftp service: FTPListFilesTask + filter: true + ftp_apply_tpl: + package: lib.plugin.base.tasks.text + service: Jinja2TemplateTask + path: lib/plugin/iscan/templates/ftp.tpl logging: - Storage: INFO \ No newline at end of file + Storage: DEBUG + Loader: DEBUG \ No newline at end of file diff --git a/lib/data/Item.py b/lib/data/Item.py new file mode 100644 index 0000000..5d74e38 --- /dev/null +++ b/lib/data/Item.py @@ -0,0 +1,17 @@ +class Item(object): + """Base class for item""" + def __init__(self, source): + self._item = { + 'source': source, + 'steps': {}, + 'data': {} + } + + def set(self, key, value): + elem = self._item['data'] + upd = {} + for x in key.split("."): + elem = elem.get(x, {}) + upd[x] = {} + upd[0] = value + self._item['data'].update(upd) \ No newline at end of file diff --git a/lib/data/Storage.py b/lib/data/Storage.py index 7193415..3bdaf10 100644 --- a/lib/data/Storage.py +++ b/lib/data/Storage.py @@ -58,6 +58,21 @@ class Storage(Loadable): def find(self): self._logger.debug("find") return self._find() + + def _update(self, items, update): + pass + + def update(self, items, update=None): + if items: + items = [i for i in items if i is not None] + self._logger.debug("update %s, %s", len(items), update) + self._update(items, update) + + def _remove(self, items): + pass + + def remove(self, items): + self._remove(items) class LiFoStorage(Storage): diff --git a/lib/data/__init__.py b/lib/data/__init__.py index dbfb839..e591024 100644 --- a/lib/data/__init__.py +++ b/lib/data/__init__.py @@ -1,7 +1,6 @@ from .Storage import Storage from .Source import Source from .Feed import Feed +from .Item import Item -from .Manager import DataManager - -__all__ = ['Storage', 'Source', 'DataManager'] +__all__ = ['Storage', 'Source', 'Feed', 'Item'] diff --git a/lib/exeq/Executor.py b/lib/exeq/Executor.py index 96ef463..6b111de 100644 --- a/lib/exeq/Executor.py +++ b/lib/exeq/Executor.py @@ -1,7 +1,5 @@ from lib import Service, Loader, Loadable -from lib.tasks.worker import worker - from time import sleep from rq import Queue @@ -17,43 +15,48 @@ class Executor(Service): class RQExecutor(Executor): - """rq (redis queue) executor - lightweight; workers placed on different nodes""" + """rq (redis queue) executor""" def __init__(self, id, root): super().__init__(self.__run, id, root) def __run(self): redis_conn = Redis(host=self.lcnf.get('redis').get('host')) - jobs = [] + while self._running: - sleep(self.lcnf.get('delay', 0.07)) + sleep(self.lcnf.get('delay', 0.2)) try: - for job in [j for j in jobs if j.result is not None]: - self._logger.debug('Publishing finished job result') - self._data.put(job.result) - job.cleanup() - jobs.remove(job) for pn, pipeline in self.cnf.get("pipelines").items(): - self._logger.debug("pipeline: %s", pn) + if pn not in self.cnf.get('core').get('pipelines'): + continue source = Loader.by_id('storage', pipeline.get('source')) for step in pipeline['steps']: - self._logger.debug("task name: %s", step['task']) q = Queue(step.get('priority', 'normal'), connection=redis_conn) - if q.count + 1 > self.lcnf.get('qsize', 100): + for job_id in jobs: + job = q.fetch_job(job_id) + if job: + if job.result is not None: + self._logger.debug("%s|%s", job_id, job._status) + self._data.update(job.result) + job.cleanup() + jobs.remove(job_id) + if len(jobs) + 1 > self.lcnf.get('qsize', 200): continue filter = {"steps.%s" % step['task']: {'$exists': False}} filter.update({key: value for key, value in step.get("if", {}).items()}) - count = step.get('multiple') if step.get('multiple', False) else 1 + count = step.get('parallel', 1) # get as much as possible from own pool items = self._data.get(block=False, count=count, filter=filter) # obtain everything else from source if len(items) < count: - items.extend(source.get(block=False, count=(count - len(items)), filter=filter)) + new_items = source.get(block=False, count=(count - len(items)), filter=filter) + items.extend(new_items) + source.remove(new_items) + if items: - for i in items: - i['steps'][step['task']] = None - self._logger.debug("enqueueing task '%s' (count: %s)", step['task'], len(items)) + self._data.update(items, {'$set': {'steps.%s' % step['task']: None}}) job = q.enqueue("lib.exeq.Task.run", step['task'], items) - jobs.append(job) + self._logger.info("%s|%s|%s|%s", job.id, step.get('priority', 'normal'), step['task'], len(items)) + jobs.append(job.id) except Exception as e: self._logger.error("Error in executor main thread: %s", e) \ No newline at end of file diff --git a/lib/plugin/Manager.py b/lib/plugin/Manager.py deleted file mode 100644 index 1c9a035..0000000 --- a/lib/plugin/Manager.py +++ /dev/null @@ -1,9 +0,0 @@ -import importlib - -class Manager: - def __init__(self): - pass - - @staticmethod - def get_plugin(name): - return importlib.import_module("lib.plugin.plugins." + name) \ No newline at end of file diff --git a/lib/plugin/README b/lib/plugin/README deleted file mode 100644 index ff9d9b4..0000000 --- a/lib/plugin/README +++ /dev/null @@ -1,6 +0,0 @@ -THIS PART IS LIKE UNDER CONSTRUCTION - -get out -for now - -come back later \ No newline at end of file diff --git a/lib/plugin/__init__.py b/lib/plugin/__init__.py deleted file mode 100644 index 89a55df..0000000 --- a/lib/plugin/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .Manager import Manager \ No newline at end of file diff --git a/lib/plugin/base/lib/IP.py b/lib/plugin/base/lib/IP.py index 6bfe074..2fd5a9c 100644 --- a/lib/plugin/base/lib/IP.py +++ b/lib/plugin/base/lib/IP.py @@ -80,12 +80,12 @@ class RandomIP(IPSource): while self._running: try: items = [] - for _ in itertools.repeat(None, self.lcnf.get("oneshot", 100)): + for _ in itertools.repeat(None, self.lcnf.get("oneshot", 200)): item = copy.deepcopy(self._item) randomip = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff))) item['data']['ip'] = str(randomip) items.append(item) self._data.put(items) - sleep(self.lcnf.get("delay", 0.5)) + sleep(self.lcnf.get("delay", 0.2)) except Exception as e: self._logger.warn(e) diff --git a/lib/plugin/base/lib/Mongo.py b/lib/plugin/base/lib/Mongo.py index 621730f..8c67204 100644 --- a/lib/plugin/base/lib/Mongo.py +++ b/lib/plugin/base/lib/Mongo.py @@ -19,26 +19,22 @@ class MongoStorage(Storage): def _get(self, block, filter): if filter is None: filter = {} - else: - self._logger.debug(filter) - item = self._coll.find_one_and_delete(filter=filter) + + item = self._coll.find_one(filter=filter) if block: while not item: - item = self._coll.find_one_and_delete(filter=filter) + item = self._coll.find_one(filter=filter) sleep(1) + return item - def _get_many(self, count, block, filter): + def _get_many(self, count, block, filter, update=None): if filter is None: filter = {} - else: - self._logger.debug(filter) + + self._logger.debug("%s, %s", filter, update) items = self._coll.find(filter=filter, limit=count) - result = [] - for i in items: - self._coll.delete_one({'_id': i['_id']}) - result.append(i) - return result - + return items + def _put(self, item, block): if block and self.size() is not 0: while self.count() + 1 > self.size(): @@ -57,3 +53,15 @@ class MongoStorage(Storage): if filter is None: filter = {} return self._coll.find(filter) + + def _update(self, items, update): + if update: + filter = {'_id': {'$in': [item['_id'] for item in items]}} + self._logger.debug("%s, %s", filter, update) + self._coll.update_many(filter, update, upsert=True) + else: + for item in items: + self._coll.replace_one({'_id': item['_id']}, item, upsert=True) + + def _remove(self, items): + self._coll.delete_many({'_id': {'$in': [item['_id'] for item in items]}}) \ No newline at end of file diff --git a/lib/plugin/base/lib/Telegram.py b/lib/plugin/base/lib/Telegram.py index ffd7e25..b684899 100644 --- a/lib/plugin/base/lib/Telegram.py +++ b/lib/plugin/base/lib/Telegram.py @@ -1,5 +1,4 @@ from lib.data import Feed, Filter -from lib.plugin import Manager import telebot from time import sleep @@ -16,24 +15,18 @@ class TelegramFeed(Feed): while self._running: try: for chat in self.lcnf.get("chats"): - chat_id = chat['id'] - sleep(delay) - continue - # plugins -> pipelines - # it is in progress - #TODO - msg = Manager.get_plugin(plugin).Plugin.TelegramMessage(host) - msg.run() - if msg.data['txt']: + chat_id = chat.get('id') + self._logger.debug(chat_id) + filter = {"feed.%s" % self._id: {'$exists': False}} + filter.update({key: value for key, value in chat.get("if", {}).items()}) + items = self._data.get(block=False, count=10, filter=filter) + self._logger.debug(items) + if items: + self._data.update(items, {'$set': {'feed.%s' % self._id: True}}) tbot = telebot.TeleBot(self.lcnf.get('token'), threaded=False) - if msg.data['img']: - self._logger.debug("Send IP with img %s:%s to %s" % (host['ip'], host['port'], chat_id)) - tbot.send_photo("@" + chat_id, msg.data['img'], caption=msg.data['txt']) - else: - self._logger.debug("Send IP %s:%s to %s" % (host['ip'], host['port'], chat_id)) - tbot.send_message("@" + chat_id, msg.data['txt']) - else: - self._logger.error('Empty text!') + for i in items: + self._logger.debug("@%s: %s", chat_id, i['data']['message']) + tbot.send_message("@" + chat_id, i['data']['message']) sleep(delay) except Exception as e: self._logger.warn(e) diff --git a/lib/plugin/base/tasks/text.py b/lib/plugin/base/tasks/text.py new file mode 100644 index 0000000..08eeffc --- /dev/null +++ b/lib/plugin/base/tasks/text.py @@ -0,0 +1,17 @@ +from lib.exeq import Task +from jinja2 import Environment, FileSystemLoader + + +class Jinja2TemplateTask(Task): + def __init__(self, id, root): + super().__init__(id, root) + + def _process(self, item): + template = Environment(loader=FileSystemLoader('.')).get_template(self.lcnf.get('path')) + item['data']['message'] = template.render(data = item['data']) + item['steps'][self._id] = True + + def _run(self, items): + for item in items: + self._process(item) + return items diff --git a/lib/plugin/iscan/plugin.yaml b/lib/plugin/iscan/plugin.yaml deleted file mode 100644 index 7845be1..0000000 --- a/lib/plugin/iscan/plugin.yaml +++ /dev/null @@ -1,62 +0,0 @@ -plugin: - name: iscan - version: 0.1 - pipelines: - FTP: - actions: - - scan - - connect - - metadata - - filetree - df: - Telegram: - action: metadata - chats: - - xai7poozengee2Aen3poMookohthaZ - - aiWeipeighah7vufoHa0ieToipooYe - HTTP: - actions: - - scan - - connect - - metadata - - screenshot - df: - Telegram: - action: screenshot - chats: - - xai7poozengee2Aen3poMookohthaZ - - gohquooFee3duaNaeNuthushoh8di2 - Gopher: - actions: - - connect - - collect - df: - Telegram: - action: collect - chats: - - xai7poozengee2Aen3poMookohthaZ - - ohl7AeGah5uo8cho4nae9Eemaeyae3 - - -df: - Telegram: - token: TOKEN - -ds: - IPRange: - file: file - Remote: - - -docker: - services: - selenium: - image: selenium/standalone-chrome:latest - volumes: - - /dev/shm:/dev/shm - environment: - - JAVA_OPTS=-Dselenium.LOGGER.level=WARNING - worker_env: - - SELENIUM_IP=selenium - required_by: - - HTTP \ No newline at end of file diff --git a/lib/plugin/iscan/tasks/common.py b/lib/plugin/iscan/tasks/common.py index 5802077..c2de352 100644 --- a/lib/plugin/iscan/tasks/common.py +++ b/lib/plugin/iscan/tasks/common.py @@ -70,6 +70,4 @@ class MasScanTask(Task): item['steps'][self._id] = result if result: self._logger.debug("Found %s with open %s", item['data']['ip'], item['data']['ports']) - - self._logger.debug(items) return items diff --git a/lib/plugin/iscan/tasks/ftp.py b/lib/plugin/iscan/tasks/ftp.py index ed2929e..cbf8e24 100644 --- a/lib/plugin/iscan/tasks/ftp.py +++ b/lib/plugin/iscan/tasks/ftp.py @@ -81,6 +81,7 @@ class FTPListFilesTask(Task): super().__init__(id, root) def _process(self, item): + item['steps'][self._id] = False self.ftp = ftplib.FTP(host=item['data']['ip'], user=item['data']['username'], passwd=item['data']['password']) @@ -100,6 +101,7 @@ class FTPListFilesTask(Task): item['data']['files'] = [] for fileName in filelist: item['data']['files'].append(fileName) + item['steps'][self._id] = True def _filter(self, item): item['data']['filter'] = False @@ -112,6 +114,8 @@ class FTPListFilesTask(Task): match += 1 if match == len(item['data']['files']): item['data']['filter'] = "EmptyWithSystemDirs" + if item['data']['filter'] == False: + item['steps'][self._id] = True def _run(self, items): for item in items: diff --git a/lib/plugin/iscan/tasks/gopher.py b/lib/plugin/iscan/tasks/gopher.py new file mode 100644 index 0000000..73255bb --- /dev/null +++ b/lib/plugin/iscan/tasks/gopher.py @@ -0,0 +1,50 @@ +import socket + +from Config import cnf + +from lib.exeq import Task + +class GopherFindTask(Task): + def __init__(self, id, root): + super().__init__(id, root) + + def _run(self, items): + for item in items: + self._process(item) + return items + + def _recv(self, sck): + total_data = [] + while True: + data = sck.recv(2048) + if not data: + break + total_data.append(data.decode('utf-8')) + return ''.join(total_data) + + def _process(self, item): + sock = socket.socket() + sock.settimeout(self.lcnf.get('timeout', 20)) + sock.connect((item['data']['ip'], int(70))) + sock.sendall(b'\n\n') + + response = self._recv(sock) + sock.close() + + self._logger.debug("Parsing result") + item['data']['files'] = [] + item['data']['filter'] = False + for s in [s for s in response.split("\r\n") if s]: + node = {} + fields = s.split("\t") + self._logger.debug(fields) + node['type'] = fields[0][0] + if len(fields) == 4: + node['name'] = fields[0][1:] + node['path'] = fields[1] + node['serv'] = f"{fields[2]}:{fields[3]}" + item['data']['files'].append(node) + + if not item['data']['files']: + item['steps'][self._id] = False + raise Exception("Empty server (not Gopher?)") diff --git a/lib/plugin/iscan/templates/ftp.tpl b/lib/plugin/iscan/templates/ftp.tpl new file mode 100644 index 0000000..f00fc97 --- /dev/null +++ b/lib/plugin/iscan/templates/ftp.tpl @@ -0,0 +1,6 @@ +ftp://{{data['username']}}:{{data['password']}}@{{data['ip']}} +{% for filename in data['files'] -%} + + {{ filename }} +{% endfor -%} +Geo: {{data['geo']['country']}}/{{data['geo']['city']}} + diff --git a/lib/plugin/iscan/templates/gopher.tpl b/lib/plugin/iscan/templates/gopher.tpl new file mode 100644 index 0000000..b756f8b --- /dev/null +++ b/lib/plugin/iscan/templates/gopher.tpl @@ -0,0 +1,11 @@ +gopher://{{data['ip']}}/ +Dirs: +{% for dir in [f for f in self._host['data']['files'] if f['type'] == '1'] -%} + + {{dir['path']}} +{% endfor -%} +Other nodes: +{% for file in [f for f in self._host['data']['files'] if f['type'] != '1' and f['type'] != 'i'] -%} + + {{file['path']}} + {{file['name']}} +{% endfor -%} +Geo: {{data['geo']['country']}}/{{data['geo']['city']}} \ No newline at end of file diff --git a/lib/plugin/plugins/FTP.py b/lib/plugin/plugins/FTP.py deleted file mode 100644 index b262554..0000000 --- a/lib/plugin/plugins/FTP.py +++ /dev/null @@ -1,18 +0,0 @@ -import ftplib -import netaddr - -from Config import cnf - - -class Plugin(): - class TelegramMessage(): - def _init(self): - self._name = "FTP" - - def _generate(self): - self.data['txt'] = "ftp://%s:%s@%s/\n" % \ - (self._host['data']['username'], self._host['data']['password'], self._host['ip']) - for filename in self._host['data']['files']: - self.data['txt'] += " + %s\n" % filename - self.data['txt'] += "Geo: %s/%s\n" % (self._host['data']['geo']['country'], self._host['data']['geo']['city']) - self.data['txt'] += "#ftp_" + str(int(netaddr.IPAddress(self._host['ip']))) diff --git a/lib/plugin/plugins/Gopher.py b/lib/plugin/plugins/Gopher.py index 4fc6f5b..28818a4 100644 --- a/lib/plugin/plugins/Gopher.py +++ b/lib/plugin/plugins/Gopher.py @@ -1,4 +1,3 @@ -import socket import netaddr from Config import cnf @@ -11,15 +10,6 @@ class Plugin(BasePlugin): self._name = "Gopher" def _generate(self): - self.data['txt'] = "gopher://%s/\n" % self._host['ip'] - self.data['txt'] += "Dirs:\n" - for dir in [f for f in self._host['data']['files'] if f['type'] == '1']: - self.data['txt'] += " + %s\n" % dir['path'] - self.data['txt'] += "Other nodes:\n" - for file in [f for f in self._host['data']['files'] if f['type'] != '1' and f['type'] != 'i']: - self.data['txt'] += " + %s\n %s\n" % (file['path'], file['name']) - self.data['txt'] += "Geo: %s/%s\n" % (self._host['data']['geo']['country'], self._host['data']['geo']['city']) - self.data['txt'] += "#gopher_" + str(int(netaddr.IPAddress(self._host['ip']))) class Pipeline(BasePlugin.Pipeline): def _init(self): @@ -33,38 +23,3 @@ class Plugin(BasePlugin): self._logger.debug("Error occured: %s (%s)", e, self._host['ip']) else: self._logger.info("Succeeded for %s" % self._host['ip']) - - def _recv(self, sck): - total_data = [] - while True: - data = sck.recv(2048) - if not data: - break - total_data.append(data.decode('utf-8')) - return ''.join(total_data) - - def _find(self): - sock = socket.socket() - sock.settimeout(cnf.stalker.Gopher.timeout) - sock.connect((self._host['ip'], int(self._host['port']))) - sock.sendall(b'\n\n') - - response = self._recv(sock) - sock.close() - - self._logger.debug("Parsing result") - self._host['data']['files'] = [] - self._host['data']['filter'] = False - for s in [s for s in response.split("\r\n") if s]: - node = {} - fields = s.split("\t") - self._logger.debug(fields) - node['type'] = fields[0][0] - if len(fields) == 4: - node['name'] = fields[0][1:] - node['path'] = fields[1] - node['serv'] = f"{fields[2]}:{fields[3]}" - self._host['data']['files'].append(node) - - if not self._host['data']['files']: - raise self.PipelineError("Empty server (not Gopher?)") diff --git a/lib/plugin/plugins/README b/lib/plugin/plugins/README deleted file mode 100644 index 305d454..0000000 --- a/lib/plugin/plugins/README +++ /dev/null @@ -1 +0,0 @@ -plugins formed in old plugin format \ No newline at end of file diff --git a/lib/plugin/plugins/__init__.py b/lib/plugin/plugins/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/plugin/test.py b/lib/plugin/test.py deleted file mode 100644 index a67f973..0000000 --- a/lib/plugin/test.py +++ /dev/null @@ -1,45 +0,0 @@ -from threading import Thread -from time import sleep - -class A: # NOOP - def __init__(self, thread = None): - if thread: - self.__thread = Thread(target=thread) - self._running = False - self._init() - - def _init(self): - pass - - def start(self): - self._running = True - self.__thread.daemon = True - self.__thread.start() - - def stop(self): - self._running = False - self.__thread.join() - - def __run(self): - while(self._running): - print('NOOP') - sleep(1) - -class B(A): # NOOP - def __init__(self): - super().__init__(self.__run) - - def __run(self): - while(self._running): - print('OP') - sleep(1) - -class C(A): # NOOP - def __run(self): - while(self._running): - print('OP') - sleep(1) - - def _init(self): - self.__thread = Thread(target=self.__run) - diff --git a/lib/util.py b/lib/util.py index 3b0c0a1..0ff606f 100644 --- a/lib/util.py +++ b/lib/util.py @@ -96,5 +96,7 @@ class Loader: @classmethod def by_id(cls, section, id): - l = cls(config[section][id].get('package')) - return l.get(config[section][id].get('service'))(id=id, root=config[section]) + # prepares Loader for certain package + l = cls(config.get(section).get(id).get('package')) + # loads class from this package and returns instantiated object of this class + return l.get(config.get(section).get(id).get('service'))(id=id, root=config.get(section))