added bunch of symbols

This commit is contained in:
Good Evening 2018-04-03 02:47:50 +03:00
parent d782c2eb70
commit 81b57337f6
22 changed files with 55 additions and 5 deletions

View File

@ -4,6 +4,8 @@ from lib import Logger, Loader, Loadable
from Config import cnf
class Service(Loadable):
"""Base class for loadale service"""
# service consists of running thread and storage attached
def __init__(self, thread, id, root=cnf):
super().__init__(id, root)

View File

@ -8,6 +8,7 @@ from lib import Service
class Feed(Service):
"""Base class for datafeeds"""
def __init__(self, thread, id, root):
super().__init__(thread, id, root)
self._logger.add_field('service', 'Feed')

View File

@ -1,6 +1,7 @@
from lib import Loader
class Filter:
"""Base class for filters..? ehm, it should be part of storage, i think"""
def __init__(self):
self._not_exist = []
self._exist = []

View File

@ -3,6 +3,7 @@ from time import sleep
from lib import Service, Loader
class DataManager(Service):
"""Actually, we may load feeds, sources and datapools right in core. Not sure that datamanager is required just to pull sources"""
def __init__(self, id, root):
super().__init__(self.__run, id, root)
self._logger.add_field('service', 'DataManager')

View File

@ -1,6 +1,7 @@
from lib import Service
class Source(Service):
"""Base class for datasources"""
def __init__(self, thread, id, root):
super().__init__(thread, id, root)
self._logger.add_field('service', 'Feed')

View File

@ -3,6 +3,7 @@ from queue import LifoQueue, Empty, Full
from lib import Loadable, Logger
class Storage(Loadable):
"""Base class for storages"""
def __init__(self, id, root):
super().__init__(id, root)

View File

@ -1,5 +1,7 @@
from lib import Loadable, Logger
# dunno
class Type(Loadable):
def __init__(self):
self.data = {}

View File

@ -7,6 +7,7 @@ from redis import Redis
class Executor(Service):
"""Base class for executors"""
def __init__(self, thread, id, root):
super().__init__(thread, id, root)
self._logger.add_field('service', 'Executor')
@ -14,12 +15,14 @@ class Executor(Service):
class RQExecutor(Executor):
"""rq (redis queue) executor - lightweight; workers placed on different nodes"""
def __init__(self, id, root):
super().__init__(self.__run, id, root)
def __run(self):
while self._running:
try:
#TODO
for pn, pipeline in self.cnf.pipelines:
self._logger.debug("pipeline: %s", pn)
for step in pipeline.steps:

View File

@ -1,5 +1,5 @@
from lib import Loadable
#TODO dunno
class Pipeline(Loadable):
def __init__(self, id, root):
super().__init__(id, root)

View File

@ -1,4 +1,7 @@
class Action:
"""Base class for remote actions"""
#this part should be removed, I think
#dunno
def __init__(self, datapool):
self._datapool = datapool
self.result = None

View File

@ -1,17 +1,21 @@
# pylint: disable=E1101
# ^
# fags
# v
import zmq
from lib import Service
from lib.net import ActionManager, Message
import zmq
class Listener(Service):
"""Base class for listeners"""
def __init__(self, thread, id, root):
super().__init__(thread, id, root)
self._logger.add_field('service', 'Listener')
self._logger.add_field('vname', self.__class__.__name__)
class ZMQListener(Listener):
"""ZMQ (Zero MQ) listener - uses my own shitty legacy proto"""
def __init__(self, id, root):
super().__init__(self.__run, id, root)

View File

@ -1,6 +1,7 @@
import pickle
class Message:
"""Message. Just a pickled dict."""
def __init__(self, data=None):
if data is None:
data = {}

View File

@ -1,5 +1,7 @@
# pylint: disable=E1101
# ^
# fags
# v
import zmq
import threading
@ -7,7 +9,11 @@ from Config import cnf
from lib.net import Message
from lib import Logger
"""This should be reworked to loadable format. Protos may be different, just like listeners"""
class Remote:
"""Used to run remote Actions.. meh"""
def __init__(self, ip, port):
self._send_lock = threading.Lock()
self._ip = ip

6
lib/plugin/README Normal file
View File

@ -0,0 +1,6 @@
THIS PART IS LIKE UNDER CONSTRUCTION
get out
for now
come back later

View File

@ -1,4 +1,5 @@
class Task:
"""Pipelines should consist of tasks??..."""
def __init__(self):
self._data = None

View File

@ -3,6 +3,7 @@ from time import sleep
from lib.data import Storage
class MongoStorage(Storage):
"""Mongo storage. Currently the only working correctly."""
def __init__(self, id, root):
super().__init__(id, root)
self._client = MongoClient(self.lcnf.url)

View File

@ -3,6 +3,7 @@ from lib.net import Remote
from time import sleep
class RemoteFeed(Feed):
"""Puts processed data to remote medved"""
def __init__(self, id, root):
super().__init__(self.__run, id, root)
@ -21,6 +22,7 @@ class RemoteFeed(Feed):
self._logger.warn(e)
class RemoteSource(Source):
"""Takes data from remote medved"""
def __init__(self, id, root):
super().__init__(self.__run, id, root)

View File

@ -6,6 +6,8 @@ from time import sleep
class TelegramFeed(Feed):
"""Send data to Telegram chat"""
# it doesn't work, though
def __init__(self, id, root):
super().__init__(self.__run, id, root)
@ -16,6 +18,9 @@ class TelegramFeed(Feed):
chat_id = chat.id
sleep(self.lcnf.delay)
continue
# plugins -> pipelines
# it is in progress
#TODO
msg = Manager.get_plugin(plugin).Plugin.TelegramMessage(host)
msg.run()
if msg.data['txt']:

View File

@ -0,0 +1,4 @@
plugins formed in old plugin format
however, currently there is no other format

View File

@ -1,6 +1,7 @@
from lib.plugin import Manager
# legacy
def worker(host, plugin):
p = Manager.get_plugin(plugin)
p.Plugin.Pipeline(host, plugin)
del p
# cool bro

View File

@ -6,6 +6,7 @@ import importlib
import sys, os
class Logger(logging.Logger):
"""Logger. standard logging logger with some shitcode on the top"""
def __init__(self, name):
self._lf_start = '[%(asctime)s][%(levelname)-7s][%(name)-10s]'
self._lf_end = ' %(message)s'
@ -71,6 +72,7 @@ class Logger(logging.Logger):
class Loadable:
"""parent for loadable from configuration"""
def __init__(self, id, root=config):
self.cnf = config
self.lcnf = root[id]
@ -78,6 +80,7 @@ class Loadable:
class Loader:
"""Loads classes by configuration"""
def __init__(self, path):
self._path = path
self._name = path.split('.')[-1]

View File

@ -7,6 +7,7 @@ from Config import cnf
from lib import Logger, Loader
class Core:
"""Core class, contains core services (like listeners, executors, datapool)"""
def __init__(self):
self.cnf = cnf.Core
self.logger = Logger("Core")