2019-07-17 12:25:10 +00:00
|
|
|
# Medved
|
|
|
|
This is extensible network scanner (Masscan-based).
|
2018-04-02 22:41:10 +00:00
|
|
|
|
2019-07-17 12:27:20 +00:00
|
|
|
## Requirements
|
|
|
|
* `docker >=18.09`
|
2019-07-17 12:28:15 +00:00
|
|
|
* `docker-compose >=1.24`
|
2019-07-17 12:27:20 +00:00
|
|
|
|
|
|
|
## Building Docker image
|
|
|
|
|
|
|
|
Run `make base && docker-compose up --build --scale worker=5`
|
|
|
|
|
|
|
|
## Running
|
|
|
|
|
|
|
|
Run `make up workers=5`
|
|
|
|
This launches a scanner with 5 workers.
|
|
|
|
|
2019-07-17 12:25:10 +00:00
|
|
|
## Configuration
|
2018-07-28 12:21:15 +00:00
|
|
|
|
|
|
|
`data/config.yaml`
|
|
|
|
```
|
2018-08-06 20:22:28 +00:00
|
|
|
---
|
|
|
|
dsl_version: 1
|
|
|
|
|
2018-07-28 12:21:15 +00:00
|
|
|
core:
|
2018-08-06 20:22:28 +00:00
|
|
|
services:
|
|
|
|
- random_ip
|
2018-07-28 12:21:15 +00:00
|
|
|
- rq_executor
|
2018-08-06 20:22:28 +00:00
|
|
|
- tg_feed
|
2018-07-28 12:21:15 +00:00
|
|
|
pipelines:
|
|
|
|
- ftp
|
2018-08-06 20:22:28 +00:00
|
|
|
- gopher
|
2018-07-28 12:21:15 +00:00
|
|
|
|
|
|
|
services:
|
2018-08-06 20:22:28 +00:00
|
|
|
random_ip:
|
|
|
|
package: lib.plugin.base.lib.IP
|
|
|
|
service: RandomIP
|
|
|
|
storage: ip_source
|
2018-07-28 12:21:15 +00:00
|
|
|
rq_executor:
|
2018-08-06 20:22:28 +00:00
|
|
|
package: lib.exec.Executor
|
2018-07-28 12:21:15 +00:00
|
|
|
service: RQExecutor
|
2018-08-06 20:22:28 +00:00
|
|
|
storage: pool
|
2018-07-28 12:21:15 +00:00
|
|
|
redis:
|
|
|
|
host: "127.0.0.1"
|
2018-08-06 20:22:28 +00:00
|
|
|
tg_feed:
|
2018-07-28 12:21:15 +00:00
|
|
|
package: lib.plugin.base.lib.Telegram
|
|
|
|
service: TelegramFeed
|
2018-08-06 20:22:28 +00:00
|
|
|
storage: pool
|
|
|
|
token: "mocken"
|
|
|
|
chats:
|
|
|
|
- id: aiWeipeighah7vufoHa0ieToipooYe
|
|
|
|
if:
|
|
|
|
steps.ftp_apply_tpl: true
|
|
|
|
data.filter: false
|
|
|
|
- id: ohl7AeGah5uo8cho4nae9Eemaeyae3
|
|
|
|
if:
|
|
|
|
steps.gopher_apply_tpl: true
|
|
|
|
data.filter: false
|
|
|
|
|
2018-07-28 12:21:15 +00:00
|
|
|
storage:
|
|
|
|
pool:
|
|
|
|
package: lib.plugin.base.lib.Mongo
|
|
|
|
service: MongoStorage
|
2018-08-06 20:22:28 +00:00
|
|
|
size: 0
|
2018-07-28 12:21:15 +00:00
|
|
|
db: "medved"
|
|
|
|
coll: 'pool'
|
2018-08-06 20:22:28 +00:00
|
|
|
ip_source:
|
2018-07-28 12:21:15 +00:00
|
|
|
package: lib.plugin.base.lib.Mongo
|
|
|
|
service: MongoStorage
|
2018-08-06 20:22:28 +00:00
|
|
|
size: 800
|
2018-07-28 12:21:15 +00:00
|
|
|
db: "medved"
|
2018-08-06 20:22:28 +00:00
|
|
|
coll: 'ip_source'
|
|
|
|
|
2018-07-28 12:21:15 +00:00
|
|
|
|
|
|
|
pipelines:
|
|
|
|
ftp:
|
2018-08-06 20:22:28 +00:00
|
|
|
source: ip_source
|
|
|
|
steps:
|
|
|
|
- task: ftp_scan
|
|
|
|
priority: low
|
|
|
|
parallel: 100
|
|
|
|
- task: ftp_connect
|
|
|
|
priority: normal
|
|
|
|
if:
|
|
|
|
steps.ftp_scan: true
|
|
|
|
- task: ftp_list_files
|
|
|
|
priority: high
|
|
|
|
if:
|
|
|
|
steps.ftp_connect: true
|
|
|
|
- task: ftp_apply_tpl
|
|
|
|
priority: high
|
|
|
|
if:
|
|
|
|
steps.ftp_list_files: true
|
|
|
|
gopher:
|
|
|
|
source: ip_source
|
|
|
|
steps:
|
|
|
|
- task: gopher_scan
|
|
|
|
priority: normal
|
|
|
|
parallel: 100
|
|
|
|
- task: gopher_find
|
|
|
|
priority: high
|
|
|
|
if:
|
|
|
|
steps.gopher_scan: true
|
|
|
|
- task: gopher_apply_tpl
|
|
|
|
priority: high
|
|
|
|
if:
|
|
|
|
steps.gopher_find: true
|
|
|
|
|
|
|
|
http:
|
|
|
|
source: ip_source
|
|
|
|
steps:
|
|
|
|
- task: http_scan
|
|
|
|
priority: low
|
|
|
|
parallel: 25
|
|
|
|
|
2018-07-28 12:21:15 +00:00
|
|
|
tasks:
|
2018-08-06 20:22:28 +00:00
|
|
|
gopher_scan:
|
|
|
|
package: lib.plugin.iscan.tasks.common
|
|
|
|
service: MasScanTask
|
|
|
|
ports:
|
|
|
|
- 70
|
|
|
|
gopher_find:
|
|
|
|
package: lib.plugin.iscan.tasks.gopher
|
|
|
|
service: GopherFindTask
|
|
|
|
gopher_apply_tpl:
|
|
|
|
package: lib.plugin.base.tasks.text
|
|
|
|
service: Jinja2TemplateTask
|
|
|
|
path: lib/plugin/iscan/templates/gopher.tpl
|
|
|
|
|
2018-07-28 12:21:15 +00:00
|
|
|
ftp_scan:
|
2018-08-06 20:22:28 +00:00
|
|
|
package: lib.plugin.iscan.tasks.common
|
|
|
|
service: MasScanTask
|
2018-07-28 12:21:15 +00:00
|
|
|
ports:
|
|
|
|
- 21
|
2018-08-06 20:22:28 +00:00
|
|
|
ftp_connect:
|
|
|
|
package: lib.plugin.iscan.tasks.ftp
|
|
|
|
service: FTPConnectTask
|
2018-07-28 12:21:15 +00:00
|
|
|
logins: data/ftp/logins.txt
|
|
|
|
passwords: data/ftp/passwords.txt
|
|
|
|
bruteforce: true
|
|
|
|
timeout: 15
|
|
|
|
ftp_list_files:
|
2018-08-06 20:22:28 +00:00
|
|
|
package: lib.plugin.iscan.tasks.ftp
|
|
|
|
service: FTPListFilesTask
|
|
|
|
filter: true
|
|
|
|
ftp_apply_tpl:
|
|
|
|
package: lib.plugin.base.tasks.text
|
|
|
|
service: Jinja2TemplateTask
|
|
|
|
path: lib/plugin/iscan/templates/ftp.tpl
|
2018-07-28 12:21:15 +00:00
|
|
|
|
|
|
|
logging:
|
2018-08-06 20:22:28 +00:00
|
|
|
Storage: DEBUG
|
|
|
|
Loader: DEBUG
|
2018-07-28 12:21:15 +00:00
|
|
|
```
|
2018-04-02 22:41:10 +00:00
|
|
|
|
2018-07-28 12:21:15 +00:00
|
|
|
## top-level services
|
|
|
|
|
2018-08-06 20:22:28 +00:00
|
|
|
### sources ###
|
|
|
|
### feeds ###
|
|
|
|
|
|
|
|
### lib.exec.Executor.RQExecutor
|
2018-07-28 12:21:15 +00:00
|
|
|
Should run pipelines described in configuration. Works via [RedisQueue](http://python-rq.org/), so needs some Redis up and running
|
|
|
|
Basically takes data from pool and submits it to workers.
|
|
|
|
RQ workers should be launched separately (`rqworker worker` from code root)
|