!53 提交安全中心代码

Merge pull request !53 from weidongkl/master
This commit is contained in:
剑子仙机 2022-03-09 17:23:00 +00:00 committed by Gitee
commit 1bd3647ed0
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
14 changed files with 677 additions and 0 deletions

View File

View File

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

View File

@ -0,0 +1,6 @@
from django.apps import AppConfig
class VulConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'apps.vul'

View File

@ -0,0 +1,83 @@
from django.db import models
from lib import BaseModel, human_datetime
from apps.host.models import HostModel
from apps.accounts.models import User
# Create your models here.
class VulAddrModel(models.Model):
vul_address = models.CharField(max_length=200)
description = models.TextField(default="")
class Meta:
db_table = "sys_vul_db"
def __str__(self):
return f'vul addresS: {self.vul_address}'
class VulBaseModel(BaseModel):
cve_id = models.CharField(max_length=100)
score = models.CharField(max_length=20, verbose_name="cve score")
description = models.TextField(default="")
pub_time = models.CharField(max_length=100, verbose_name="publish time")
vul_level = models.CharField(max_length=100)
detail = models.TextField(default="")
software_name = models.CharField(max_length=100)
fixed_time = models.CharField(max_length=100)
fixed_version = models.CharField(max_length=100)
os = models.CharField(max_length=100)
update_time = models.DateTimeField(verbose_name="", blank=True, null=True)
class Meta:
abstract = True
class VulModel(VulBaseModel):
status = models.CharField(max_length=100)
host = models.ManyToManyField(to=HostModel, verbose_name='关联主机', db_constraint=False)
class Meta:
db_table = "sys_vul"
unique_together = [['cve_id', 'software_name', 'os']]
def __str__(self):
return f'vul: {self.cve_id} os: {self.os} software: {self.software_name}'
class SecurityAdvisoryModel(VulBaseModel):
host = models.ManyToManyField(to=HostModel, verbose_name='关联主机', db_constraint=False)
class Meta:
db_table = "sys_sa"
unique_together = [['cve_id', 'software_name', 'os']]
def __str__(self):
return f'Fixed CVE with errata{self.cve_id} os: {self.os} software: {self.software_name}'
class SecurityAdvisoryFixHistoryModel(BaseModel):
cve_id = models.CharField(max_length=100)
vul_level = models.CharField(max_length=100)
fixed_at = models.CharField(max_length=20, default=human_datetime, verbose_name="修复时间")
created_by = models.ForeignKey(User, on_delete=models.SET_DEFAULT, default=None)
status = models.CharField(max_length=20, default="success", verbose_name="修复状态")
host = models.ManyToManyField(to=HostModel, verbose_name='关联主机', through="SaFixHistToHost")
class Meta:
db_table = "sys_sa_fix_hist"
unique_together = [['cve_id', 'fixed_at']]
def __str__(self):
return f'sa fix history{self.cve_id} '
class SaFixHistToHost(models.Model):
sa_fix_hist = models.ForeignKey(to=SecurityAdvisoryFixHistoryModel, on_delete=models.CASCADE)
host = models.ForeignKey(to=HostModel, on_delete=models.CASCADE)
status = models.CharField(max_length=10)
details = models.TextField(default="")
class Meta:
db_table = "sys_sa_fix_hist_host"

View File

@ -0,0 +1,98 @@
# -*- encoding: utf-8 -*-
"""
@File : ssh_pool.py
@Time : 2022/2/22 下午3:50
@Author : weidongkl
@Email : weidong@uniontech.com
@Software: PyCharm
"""
import queue
import logging
import multiprocessing
import time
from lib.ssh import SSH
class SshProcessQueueManager:
# 设置默认进程数为8
DEFAULT_FORKS = 8
def __init__(self, hosts):
self.hosts = hosts
# 获取cpu数
cpu_count = multiprocessing.cpu_count()
# 设置cpu和默认进程数中的最大值为进程池大小当任务小于进程池大小时设置进程池大小为任务数
self.forks = min(len(self.hosts), max(self.DEFAULT_FORKS, cpu_count))
def ssh_command(self, que, host, cmd):
ssh_cli = SSH(host.ip, host.port, host.username, host.private_key)
try:
with ssh_cli as ssh:
status, result = ssh.exec_command(cmd)
que.put({'host': host.hostname,
'ret': {
"status": status,
"result": result
}})
except Exception as e:
logging.error(e)
que.put({'host': host.hostname,
'ret': {
"status": 1,
"result": e
}})
def run_subprocess(self, hosts, func, *args):
que = multiprocessing.Queue()
running = {}
hosts_iter = hosts.__iter__()
success = []
fail = []
flags = True
while True:
if not hosts or len(set(success + fail)) >= len(hosts):
break
if len(running) < self.forks and flags:
try:
host = next(hosts_iter)
vars = (
que,
host,
*args
)
subprocess = multiprocessing.Process(target=func, args=vars)
subprocess.start()
running[host.hostname] = subprocess
except StopIteration:
flags = False
for host in running:
if not running[host].is_alive():
try:
while True:
result = que.get(False)
success.append(result["host"])
yield result
except queue.Empty:
pass
if host not in success:
logging.warning("get {}'s information failed".format(host))
fail.append(host)
for host in success + fail:
if host in running:
running.pop(host)
time.sleep(0.1)
def run(self, func, *args):
result = []
for i in self.run_subprocess(self.hosts, func, *args):
result.append(i)
return result
if __name__ == "__main__":
pass

View File

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

View File

@ -0,0 +1,19 @@
# -*- encoding: utf-8 -*-
"""
@File : urls.py
@Time : 2022/2/10 下午1:49
@Author : weidongkl
@Email : weidong@uniontech.com
@Software: PyCharm
"""
from django.urls import path
from apps.vul import views
urlpatterns = [
path('api/v1/vul/hist/', views.SaFixHistListView.as_view()),
path('api/v1/vul/hist/<int:pk>/<str:hostname>/', views.SaFixHistDetailHostView.as_view()),
path('api/v1/vul/hist/<int:pk>/', views.SaFixHistDetailsView.as_view()),
path('api/v1/vul/summary/', views.VulSummaryView.as_view()),
path('api/v1/vul/', views.VulListView.as_view()),
path('api/v1/vul/<str:cve_id>/', views.VulDetailsView.as_view()),
]

246
sysom_api/apps/vul/views.py Normal file
View File

@ -0,0 +1,246 @@
import logging
import re
from rest_framework.views import APIView
from apscheduler.schedulers.background import BackgroundScheduler
from django_apscheduler.jobstores import register_job
from tzlocal import get_localzone
from lib.response import *
from apps.accounts.authentication import Authentication
from apps.vul.models import *
from apps.vul.vul import update_sa as upsa, update_vul as upvul
from apps.vul.vul import fix_cve, get_unfix_cve
logger = logging.getLogger(__name__)
scheduler = BackgroundScheduler(timezone=get_localzone())
@register_job(scheduler, 'cron', id='update_vul', hour=0, minute=0)
def update_vul():
upvul()
@register_job(scheduler, 'cron', id='update_sa', hour=2, minute=0)
def update_sa():
upsa()
scheduler.start()
class VulListView(APIView):
authentication_classes = [Authentication]
def get(self, request, format=None):
"""
Return a list of all users.
"""
cves = SecurityAdvisoryModel.objects.exclude(host=None)
cves_data = {}
for cve in cves:
cve_id = cve.cve_id
if cves_data.get(cve_id):
hosts = [host[0] for host in cve.host.all().values_list("hostname")]
cves_data[cve_id]["hosts"] += hosts
else:
vul_level = cve.vul_level
score = cve.score
description = cve.description
detail = cve.detail
pub_time = cve.pub_time
hosts = [host[0] for host in cve.host.all().values_list("hostname")]
cves_data[cve_id] = {
"cve_id": cve_id,
"vul_level": vul_level,
"score": score,
"description": str(description),
"detail": detail,
"pub_time": pub_time,
"hosts": hosts,
}
data = list(cves_data.values())
return success(result=data)
def post(self, request, format=None):
logger.error(request.user)
failed = False
data = []
for cve in request.data.get("cve_id_list"):
hosts = cve["hostname"]
cve_id = cve["cve_id"]
results = fix_cve(hosts, cve_id, user=request.user)
logger.warning(results)
sucess_host_list = []
fail_host_list = []
for ret in results:
hostname = ret["host"]
if ret["ret"]["status"] == 0:
sucess_host_list.append(hostname)
else:
failed = True
fail_host_list.append({
"hosts": hostname,
"describe": str(ret["ret"]["result"])
})
data.append({
"cve_id": cve_id,
"sucess_host_list": sucess_host_list,
"fail_host_list": fail_host_list
})
logger.error(data)
if failed:
return other_response(message='fix cve failed', code=200, result=data)
else:
return success(result=data)
class VulDetailsView(APIView):
authentication_classes = [Authentication]
def get(self, request, cve_id, format=None):
"""
Return a list of all users.
"""
cve_re = "CVE-\d{4}-\d{4,7}"
if re.match(cve_re, cve_id, re.I) is None:
other_response(message='Illegal parameter', code=400)
cves = SecurityAdvisoryModel.objects.exclude(host=None).filter(cve_id=cve_id)
cves_data = {}
flag = False
for cve in cves:
cve_id = cve.cve_id
vul_level = cve.vul_level
if flag:
software_name = cve.software_name
fixed_version = cve.fixed_version
hosts = [self.get_host_info(host[0]) for host in cve.host.all().values_list("hostname")]
cves_data["software"].append({
"name": software_name,
"vul_level": vul_level,
"fixed_version": fixed_version
})
cves_data["hosts"] += hosts
else:
flag = True
score = cve.score
description = cve.description
detail = cve.detail
pub_time = cve.pub_time
software_name = cve.software_name
fixed_version = cve.fixed_version
hosts = [self.get_host_info(host[0]) for host in cve.host.all().values_list("hostname")]
cves_data = {
"cve_id": cve_id,
"vul_level": vul_level,
"score": score,
"description": str(description),
"detail": detail,
"pub_time": pub_time,
"hosts": hosts,
"software": [{
"name": software_name,
"vul_level": vul_level,
"fixed_version": fixed_version
}]
}
data = cves_data
return success(result=data)
def get_host_info(self, hostname):
host = HostModel.objects.filter(hostname=hostname).first()
return {
"hostname": hostname,
"ip": host.ip,
"created_by": host.created_by.username,
"created_at": host.created_at,
"status": host.get_status_display(),
}
class VulSummaryView(APIView):
authentication_classes = [Authentication]
def get(self, request, format=None):
"""
"""
sa = SecurityAdvisoryModel.objects.exclude(host=None)
sa_cve_count, sa_high_cve_count, sa_affect_host_count = self.get_vul_info(sa)
unfix_vul = get_unfix_cve().exclude(host=None)
vul_cve_count, vul_high_cve_count, vul_affect_host_count = self.get_vul_info(unfix_vul)
data = {
"fixed_cve": {
"affect_host_count": sa_affect_host_count,
"cve_count": sa_cve_count,
"high_cve_count": sa_high_cve_count,
},
"unfixed_cve": {
"affected_host_number": vul_affect_host_count,
"cve_number": vul_cve_count,
"high_cve_number": vul_high_cve_count,
}
}
return success(result=data)
def get_vul_info(self, queryset):
cve_count = len(set([cve[0] for cve in queryset.values_list("cve_id")]))
high_cve_count = len(set([cve[0] for cve in queryset.filter(score__gt=7.0).values_list("cve_id")]))
affect_host = []
for cve in queryset:
affect_host.extend(list(cve.host.all()))
affect_host_count = len(set(affect_host))
return cve_count, high_cve_count, affect_host_count
class SaFixHistListView(APIView):
authentication_classes = [Authentication]
def get(self, request, format=None):
sa_fix_hist = SecurityAdvisoryFixHistoryModel.objects.all()
data = [{"id": fix_obj.id,
"cve_id": fix_obj.cve_id,
"fixed_time": fix_obj.fixed_at,
"fix_user": fix_obj.created_by.username,
"status": fix_obj.status,
"vul_level": fix_obj.vul_level} for fix_obj in sa_fix_hist]
return success(result=data)
class SaFixHistDetailsView(APIView):
authentication_classes = [Authentication]
def get_cve2host_details(self, sa_fix_host_obj):
hostname = sa_fix_host_obj.host.hostname
host = HostModel.objects.filter(hostname=hostname).first()
return {
"hostname": hostname,
"ip": host.ip,
"created_by": host.created_by.username,
"created_at": host.created_at,
"host_status": host.get_status_display(),
"status": sa_fix_host_obj.status,
"details": str(sa_fix_host_obj.details),
}
def get(self, request, pk, format=None):
sa_fix_hist_details = SaFixHistToHost.objects.filter(sa_fix_hist_id=pk)
data = [self.get_cve2host_details(detail_obj) for detail_obj in sa_fix_hist_details]
for item in range(len(data)):
data[item]["id"] = item + 1
return success(result=data)
class SaFixHistDetailHostView(APIView):
authentication_classes = [Authentication]
def get(self, request, pk, hostname, format=None):
sa_fix_hist_details_host = SaFixHistToHost.objects.filter(sa_fix_hist_id=pk, host__hostname=hostname).first()
data = {
"hostname": hostname,
"status": sa_fix_hist_details_host.status,
"details": str(sa_fix_hist_details_host.details),
}
return success(result=data)

212
sysom_api/apps/vul/vul.py Normal file
View File

@ -0,0 +1,212 @@
# -*- encoding: utf-8 -*-
"""
@File : vul.py
@Time : 2022/2/16 下午3:37
@Author : weidongkl
@Email : weidong@uniontech.com
@Software: PyCharm
"""
import logging
import requests
import json
from django.utils import timezone
from rest_framework import status
from apps.vul.models import *
from apps.host.models import HostModel
from apps.vul.ssh_pool import SshProcessQueueManager
from lib.utils import human_datetime
def update_vul():
update_vul_db()
def update_vul_db():
"""
更新漏洞数据库数据
"""
logging.debug("Begin to get vul db address")
vul_addrs = VulAddrModel.objects.all()
for vul_addr in vul_addrs:
logging.info("Try to get vul db info")
resp = requests.get(vul_addr.vul_address)
if resp.status_code != status.HTTP_200_OK:
logging.warning("update vul information failed")
break
body = json.loads(resp.text)
for cve in body["data"]["items"]:
logging.info("Update sys_vul vul data")
cve_obj_search = VulModel.objects.filter(cve_id=cve['cveid'], os=str(cve['os']),
software_name=cve['source'])
if len(cve_obj_search) == 0:
VulModel.objects.create(cve_id=cve['cveid'],
score=cve['score'],
description=str(cve['description']),
pub_time=cve['pub_time'],
vul_level=cve['vul_level'],
detail=str(cve['detail']),
software_name=cve['source'],
fixed_time=str(cve['fixed_time']),
fixed_version=str(cve['fixed_version']),
os=str(cve['os']),
status=cve['status'],
update_time=timezone.now())
else:
cve_obj_search.update(
score=cve['score'],
description=str(cve['description']),
pub_time=cve['pub_time'],
vul_level=cve['vul_level'],
detail=str(cve['detail']),
software_name=cve['source'],
fixed_time=str(cve['fixed_time']),
fixed_version=str(cve['fixed_version']),
os=str(cve['os']),
status=cve['status'],
update_time=timezone.now())
def get_unfix_cve():
"""
Returns: QuerySet in django
"""
unfix_cve_obj_search = VulModel.objects.filter(status="unfix")
return unfix_cve_obj_search
def get_unfix_cve_format():
"""
获取python 对象格式的未修复cve信息
Returns: []
"""
queryset = get_unfix_cve()
data = []
for i in queryset:
data.append({"cve_id": i.cve_id,
"os": i.os,
"software_name": i.software_name})
return data
def update_sa():
cmd = 'dnf check-update cve *;cat /etc/os-release'
spqm = SshProcessQueueManager(list(HostModel.objects.all()))
results = spqm.run(spqm.ssh_command, cmd)
#
# cve2host_info={"cve1":[(host,software,version,os)]}
# [{'host': 'GqYLM32pIZaNH0rOjd7JViwxPs', 'ret': {'status': 1, 'result': timeout('timed out')}}]
cve2host_info = {}
for result in results:
host = result["host"]
if result["ret"]['status'] == 0:
cves, software, version, os = parse_sa_result(result["ret"]['result'])
for cve in cves:
if cve in cve2host_info.keys():
cve2host_info[cve].append((host, software, version, os))
else:
cve2host_info[cve] = [(host, software, version, os)]
update_sa_db(cve2host_info)
def update_sa_db(cveinfo):
#
# cve2host_info={"cve1":[(host,software,version,os)]}
#
current_cveinfo = SecurityAdvisoryModel.objects.all()
current_cves = set([cve for cve in current_cveinfo.values_list("cve_id", "software_name", "fixed_version", "os")])
new_cveinfo = cveinfo
new_cves = set([(k, item[1], item[2], item[3]) for k, v in new_cveinfo.items() for item in v])
delete_cves = current_cves - new_cves
# 删除无效的关联关系,用于更新客户手动修复漏洞后,导致的数据库不匹配问题
for cve in list(delete_cves):
cve_id, software_name, _, os = cve
sacve_obj = SecurityAdvisoryModel.objects.filter(cve_id=cve_id, software_name=software_name, os=os).first()
sacve_obj.host.clear()
add_cves = new_cves - current_cves
for cve in list(add_cves):
cve_id, software_name, fixed_version, os = cve
cve_obj_search = VulModel.objects.filter(cve_id=cve_id,
software_name=software_name)
hosts = [host[0] for host in new_cveinfo[cve_id]]
# 增加需要新增的cve列表
if len(cve_obj_search) == 0:
sacve = SecurityAdvisoryModel.objects.create(cve_id=cve_id,
software_name=software_name,
fixed_version=fixed_version,
os=os,
update_time=timezone.now())
else:
# 是用vul漏洞数据中的已知数据填充errata未获取到的数据
cve_obj = cve_obj_search.first()
sacve = SecurityAdvisoryModel.objects.create(cve_id=cve_id,
score=cve_obj.score,
description=cve_obj.description,
pub_time=cve_obj.pub_time,
vul_level=cve_obj.vul_level,
detail=cve_obj.detail,
software_name=software_name,
fixed_version=fixed_version,
os=os,
update_time=timezone.now())
# 新增漏洞关联主机
sacve.host.add(*HostModel.objects.filter(hostname__in=hosts))
update_cves = new_cves & current_cves
for cve in list(update_cves):
cve_id, software_name, _, os = cve
hosts = [host[0] for host in new_cveinfo[cve_id]]
sacve_obj = SecurityAdvisoryModel.objects.filter(cve_id=cve_id, software_name=software_name, os=os).first()
sacve_obj.host.clear()
sacve_obj.host.add(*HostModel.objects.filter(hostname__in=hosts))
def parse_sa_result(result):
"""解析dnf获取的sa数据"""
# TODO
return result
def fix_cve(hosts, cve_id, user):
cmd = 'dnf install --cve {}'.format(cve_id)
spqm = SshProcessQueueManager(list(HostModel.objects.filter(hostname__in=hosts)))
results = spqm.run(spqm.ssh_command, cmd)
fixed_time = human_datetime()
user_obj = user
vul_level = SecurityAdvisoryModel.objects.filter(cve_id=cve_id).first().vul_level
cve_status = "success"
init = True
for ret in results:
hostname = ret["host"]
host_obj = HostModel.objects.filter(hostname=hostname).first()
if ret["ret"]["status"] == 0:
status = "success"
details = ret["ret"]["result"]
sa_obj = SecurityAdvisoryModel.objects.filter(host__hostname=hostname, cve_id=cve_id).first()
sa_obj.host.remove(host_obj)
else:
status = "fail"
cve_status = "fail"
details = ret["ret"]["result"]
if init:
safh = SecurityAdvisoryFixHistoryModel.objects.create(fixed_at=fixed_time,
cve_id=cve_id,
vul_level=vul_level,
created_by=user_obj,
status=cve_status)
init = False
elif cve_status == "fail":
safh.status = cve_status
safh.save()
safh.host.add(host_obj, through_defaults={'status': status, "details": details})
return results
if __name__ == "__main__":
pass

View File

@ -0,0 +1 @@
[{"model": "vul.vuladdrmodel", "pk": 1, "fields": {"vul_address": "https://vul.uniontech.com/", "description": "uniontech vulnerability database"}}]

View File

@ -15,6 +15,7 @@ SECRET_KEY = 'django-insecure-^d8b9di9w&-mmsbpt@)o#e+2^z+^m4nhf+z8304%9@8y#ko46l
ALLOWED_HOSTS = ['*']
INSTALLED_APPS = [
'apps.vul',
'apps.accounts',
'apps.host',
'apps.monitor',
@ -28,6 +29,7 @@ INSTALLED_APPS = [
'drf_yasg', # 在线API文档
'channels',
'django_filters',
'django_apscheduler',
]
MIDDLEWARE = [

View File

@ -68,3 +68,4 @@ urllib3==1.26.7
uWSGI==2.0.19.1
webencodings==0.5.1
zope.interface==5.4.0
django-apscheduler==0.6.0

View File

@ -25,6 +25,7 @@ urlpatterns = [
path('', include("apps.task.urls")),
path('', include("apps.vmcore.urls")),
path('', include("apps.alarm.urls")),
path('', include("apps.vul.urls")),
]
if settings.DEBUG:

View File

@ -133,9 +133,11 @@ init_conf() {
python manage.py makemigrations task
python manage.py makemigrations monitor
python manage.py makemigrations alarm
python manage.py makemigrations vul
python manage.py migrate
python manage.py loaddata ./apps/accounts/user.json
python manage.py loaddata ./apps/vmcore/vmcore.json
python manage.py loaddata ./apps/vul/vuladdr.json
popd
}