diff --git a/lib/Service.py b/lib/Service.py index 46b6232..542895b 100644 --- a/lib/Service.py +++ b/lib/Service.py @@ -4,6 +4,8 @@ from lib import Logger, Loader, Loadable from Config import cnf class Service(Loadable): + """Base class for loadale service""" + # service consists of running thread and storage attached def __init__(self, thread, id, root=cnf): super().__init__(id, root) diff --git a/lib/data/Feed.py b/lib/data/Feed.py index 7c90090..17f3ebe 100644 --- a/lib/data/Feed.py +++ b/lib/data/Feed.py @@ -8,6 +8,7 @@ from lib import Service class Feed(Service): + """Base class for datafeeds""" def __init__(self, thread, id, root): super().__init__(thread, id, root) self._logger.add_field('service', 'Feed') diff --git a/lib/data/Filter.py b/lib/data/Filter.py index cc2fcf0..d77859b 100644 --- a/lib/data/Filter.py +++ b/lib/data/Filter.py @@ -1,6 +1,7 @@ from lib import Loader class Filter: + """Base class for filters..? ehm, it should be part of storage, i think""" def __init__(self): self._not_exist = [] self._exist = [] diff --git a/lib/data/Manager.py b/lib/data/Manager.py index cab9b65..7fcbff3 100644 --- a/lib/data/Manager.py +++ b/lib/data/Manager.py @@ -3,6 +3,7 @@ from time import sleep from lib import Service, Loader class DataManager(Service): + """Actually, we may load feeds, sources and datapools right in core. Not sure that datamanager is required just to pull sources""" def __init__(self, id, root): super().__init__(self.__run, id, root) self._logger.add_field('service', 'DataManager') diff --git a/lib/data/Source.py b/lib/data/Source.py index 22f74b1..c5ec8bc 100644 --- a/lib/data/Source.py +++ b/lib/data/Source.py @@ -1,6 +1,7 @@ from lib import Service class Source(Service): + """Base class for datasources""" def __init__(self, thread, id, root): super().__init__(thread, id, root) self._logger.add_field('service', 'Feed') diff --git a/lib/data/Storage.py b/lib/data/Storage.py index 7480c12..8c565a9 100644 --- a/lib/data/Storage.py +++ b/lib/data/Storage.py @@ -3,6 +3,7 @@ from queue import LifoQueue, Empty, Full from lib import Loadable, Logger class Storage(Loadable): + """Base class for storages""" def __init__(self, id, root): super().__init__(id, root) diff --git a/lib/data/Type.py b/lib/data/Type.py index 58af0d0..f577c99 100644 --- a/lib/data/Type.py +++ b/lib/data/Type.py @@ -1,5 +1,7 @@ from lib import Loadable, Logger +# dunno + class Type(Loadable): def __init__(self): self.data = {} diff --git a/lib/exeq/Executor.py b/lib/exeq/Executor.py index de2e078..4c88896 100644 --- a/lib/exeq/Executor.py +++ b/lib/exeq/Executor.py @@ -7,6 +7,7 @@ from redis import Redis class Executor(Service): + """Base class for executors""" def __init__(self, thread, id, root): super().__init__(thread, id, root) self._logger.add_field('service', 'Executor') @@ -14,12 +15,14 @@ class Executor(Service): class RQExecutor(Executor): + """rq (redis queue) executor - lightweight; workers placed on different nodes""" def __init__(self, id, root): super().__init__(self.__run, id, root) def __run(self): while self._running: try: + #TODO for pn, pipeline in self.cnf.pipelines: self._logger.debug("pipeline: %s", pn) for step in pipeline.steps: diff --git a/lib/exeq/Pipeline.py b/lib/exeq/Pipeline.py index 34b3158..84ea0fe 100644 --- a/lib/exeq/Pipeline.py +++ b/lib/exeq/Pipeline.py @@ -1,5 +1,5 @@ from lib import Loadable - +#TODO dunno class Pipeline(Loadable): def __init__(self, id, root): super().__init__(id, root) diff --git a/lib/net/Action.py b/lib/net/Action.py index ea5c5e9..ae31716 100644 --- a/lib/net/Action.py +++ b/lib/net/Action.py @@ -1,4 +1,7 @@ class Action: + """Base class for remote actions""" + #this part should be removed, I think + #dunno def __init__(self, datapool): self._datapool = datapool self.result = None diff --git a/lib/net/Listener.py b/lib/net/Listener.py index 31d707e..3ce80f4 100644 --- a/lib/net/Listener.py +++ b/lib/net/Listener.py @@ -1,17 +1,21 @@ # pylint: disable=E1101 - +# ^ +# fags +# v +import zmq from lib import Service from lib.net import ActionManager, Message -import zmq class Listener(Service): + """Base class for listeners""" def __init__(self, thread, id, root): super().__init__(thread, id, root) self._logger.add_field('service', 'Listener') self._logger.add_field('vname', self.__class__.__name__) class ZMQListener(Listener): + """ZMQ (Zero MQ) listener - uses my own shitty legacy proto""" def __init__(self, id, root): super().__init__(self.__run, id, root) diff --git a/lib/net/Message.py b/lib/net/Message.py index 69e92b2..3d446ff 100644 --- a/lib/net/Message.py +++ b/lib/net/Message.py @@ -1,6 +1,7 @@ import pickle class Message: + """Message. Just a pickled dict.""" def __init__(self, data=None): if data is None: data = {} diff --git a/lib/net/Remote.py b/lib/net/Remote.py index 81c2f76..50b3976 100644 --- a/lib/net/Remote.py +++ b/lib/net/Remote.py @@ -1,5 +1,7 @@ # pylint: disable=E1101 - +# ^ +# fags +# v import zmq import threading @@ -7,7 +9,11 @@ from Config import cnf from lib.net import Message from lib import Logger +"""This should be reworked to loadable format. Protos may be different, just like listeners""" + + class Remote: + """Used to run remote Actions.. meh""" def __init__(self, ip, port): self._send_lock = threading.Lock() self._ip = ip diff --git a/lib/plugin/README b/lib/plugin/README new file mode 100644 index 0000000..ff9d9b4 --- /dev/null +++ b/lib/plugin/README @@ -0,0 +1,6 @@ +THIS PART IS LIKE UNDER CONSTRUCTION + +get out +for now + +come back later \ No newline at end of file diff --git a/lib/plugin/Task.py b/lib/plugin/Task.py index d5026e4..7ee39b7 100644 --- a/lib/plugin/Task.py +++ b/lib/plugin/Task.py @@ -1,4 +1,5 @@ class Task: + """Pipelines should consist of tasks??...""" def __init__(self): self._data = None diff --git a/lib/plugin/base/lib/Mongo.py b/lib/plugin/base/lib/Mongo.py index 3be0bee..3660383 100644 --- a/lib/plugin/base/lib/Mongo.py +++ b/lib/plugin/base/lib/Mongo.py @@ -3,6 +3,7 @@ from time import sleep from lib.data import Storage class MongoStorage(Storage): + """Mongo storage. Currently the only working correctly.""" def __init__(self, id, root): super().__init__(id, root) self._client = MongoClient(self.lcnf.url) diff --git a/lib/plugin/base/lib/Remote.py b/lib/plugin/base/lib/Remote.py index 3280b51..1df0e2e 100644 --- a/lib/plugin/base/lib/Remote.py +++ b/lib/plugin/base/lib/Remote.py @@ -3,6 +3,7 @@ from lib.net import Remote from time import sleep class RemoteFeed(Feed): + """Puts processed data to remote medved""" def __init__(self, id, root): super().__init__(self.__run, id, root) @@ -21,6 +22,7 @@ class RemoteFeed(Feed): self._logger.warn(e) class RemoteSource(Source): + """Takes data from remote medved""" def __init__(self, id, root): super().__init__(self.__run, id, root) diff --git a/lib/plugin/base/lib/Telegram.py b/lib/plugin/base/lib/Telegram.py index 8181198..35b8338 100644 --- a/lib/plugin/base/lib/Telegram.py +++ b/lib/plugin/base/lib/Telegram.py @@ -6,6 +6,8 @@ from time import sleep class TelegramFeed(Feed): + """Send data to Telegram chat""" + # it doesn't work, though def __init__(self, id, root): super().__init__(self.__run, id, root) @@ -16,6 +18,9 @@ class TelegramFeed(Feed): chat_id = chat.id sleep(self.lcnf.delay) continue + # plugins -> pipelines + # it is in progress + #TODO msg = Manager.get_plugin(plugin).Plugin.TelegramMessage(host) msg.run() if msg.data['txt']: diff --git a/lib/plugin/plugins/README b/lib/plugin/plugins/README new file mode 100644 index 0000000..697c45e --- /dev/null +++ b/lib/plugin/plugins/README @@ -0,0 +1,4 @@ +plugins formed in old plugin format + +however, currently there is no other format + diff --git a/lib/tasks/worker.py b/lib/tasks/worker.py index 3a9eec8..1f18157 100644 --- a/lib/tasks/worker.py +++ b/lib/tasks/worker.py @@ -1,6 +1,7 @@ from lib.plugin import Manager - +# legacy def worker(host, plugin): p = Manager.get_plugin(plugin) p.Plugin.Pipeline(host, plugin) del p + # cool bro diff --git a/lib/util.py b/lib/util.py index 3a34a2e..d1c6fb1 100644 --- a/lib/util.py +++ b/lib/util.py @@ -6,6 +6,7 @@ import importlib import sys, os class Logger(logging.Logger): + """Logger. standard logging logger with some shitcode on the top""" def __init__(self, name): self._lf_start = '[%(asctime)s][%(levelname)-7s][%(name)-10s]' self._lf_end = ' %(message)s' @@ -71,6 +72,7 @@ class Logger(logging.Logger): class Loadable: + """parent for loadable from configuration""" def __init__(self, id, root=config): self.cnf = config self.lcnf = root[id] @@ -78,6 +80,7 @@ class Loadable: class Loader: + """Loads classes by configuration""" def __init__(self, path): self._path = path self._name = path.split('.')[-1] diff --git a/medved.py b/medved.py index 0722e24..0385ec1 100755 --- a/medved.py +++ b/medved.py @@ -7,6 +7,7 @@ from Config import cnf from lib import Logger, Loader class Core: + """Core class, contains core services (like listeners, executors, datapool)""" def __init__(self): self.cnf = cnf.Core self.logger = Logger("Core")