maxjhandsome
/
sysom
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!89 优化日志中心，支持搜索 

* 优化日志中心，支持搜索；
* 优化节点初始化和清除操作；
This commit is contained in:
剑子仙机 2022-03-21 09:00:20 +00:00
parent fe003c85fb
commit b64fcfeb4d
10 changed files with 211 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sysom_api/apps/accounts/models.py
View File

@ -79,6 +79,7 @@ class HandlerLog(BaseModel):
class Meta:
db_table = 'sys_handler_log'
verbose_name_plural = verbose_name = '操作日志'
ordering = ['-created_at']
def __str__(self):
return f'{self.user.username}: 于{self.created_at} 通过{self.request_method}方式请求{self.request_url},' \

1
sysom_api/apps/accounts/urls.py
View File

@ -13,5 +13,6 @@ urlpatterns = [
path('api/v1/', include(router.urls)),
path('api/v1/auth/', views.AuthAPIView.as_view()),
path('api/v1/journal/', views.UserModelViewSet.as_view({'get': 'get_logs'})),
path('api/v1/response_code/', views.UserModelViewSet.as_view({'get': 'get_response_code'})),
path('api/v1/change_password/', views.ChangePasswordViewSet.as_view())
]

48
sysom_api/apps/accounts/views.py
View File

@ -1,7 +1,9 @@
from datetime import datetime
import logging
from rest_framework.request import Request
from rest_framework.viewsets import GenericViewSet
from rest_framework import mixins, status
from rest_framework.status import *
from rest_framework.views import APIView
from rest_framework.generics import CreateAPIView
@ -70,21 +72,51 @@ class UserModelViewSet(
result = super().retrieve(request, *args, **kwargs)
return success(result=result.data, message="获取成功")
def get_logs(self, request, *args, **kwargs):
params = request.query_params.dict()
option = self.logging_options.get(params.get('option'), None)
queryset = models.HandlerLog.objects.select_related().all()
def get_logs(self, request):
queryset = self._filter_log_params(request, models.HandlerLog.objects.select_related().all())
user = getattr(request, 'user', None)
if not user.is_admin:
queryset = queryset.filter(user=user)
if option is not None:
queryset = queryset.filter(request_option=option)
page = self.paginate_queryset(queryset)
if page is not None:
ser = serializer.HandlerLoggerListSerializer(page, many=True)
return self.get_paginated_response(ser.data)
ser = serializer.HandlerLoggerListSerializer(queryset, many=True)
return success(result=ser.data)
def _filter_log_params(self, request, queryset):
kwargs = dict()
params = request.query_params.dict()
request_option = params.pop('request_option', None)
if request_option:
option = self.logging_options.get(request_option, None)
if option is not None:
kwargs['request_option'] = option
request_ip = params.get('request_ip', None)
request_url = params.get('request_url', None)
request_method: str = params.get('request_method', None)
response_status = params.get('response_status', None)
start_time = params.get('startTime', '2000-01-01 00:00:00')
end_time = params.get('endTime', datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
if request_ip:
kwargs['request_ip'] = request_ip
if request_url:
kwargs['request_url'] = request_url
if request_method:
kwargs['request_method'] = request_method
if response_status:
kwargs['response_status'] = response_status
queryset = queryset.filter(created_at__range=[start_time, end_time], **kwargs)
return queryset
def get_response_code(self, request):
status_map = [{'label': k, 'value': v}for k, v in globals().items() if k.startswith('HTTP')]
return success(result=status_map)
class AuthAPIView(CreateAPIView):
authentication_classes = []

42
sysom_api/apps/host/views.py
View File

@ -15,6 +15,7 @@ from django.conf import settings
from apps.host import serializer
from apps.host.models import HostModel, Cluster
from apps.accounts.authentication import Authentication
from apps.task.views import script_task
from consumer.executors import SshJob
from apps.task.models import JobModel
from lib import *
@ -59,7 +60,8 @@ class HostModelViewSet(GenericViewSet,
self.perform_create(create_serializer)
instance = create_serializer.instance
# 检查输入client部署命令 更新host状态
self.client_deploy_cmd_execute(instance, 'init')
thread = threading.Thread(target=self.client_deploy_cmd_init, args=(instance,))
thread.start()
host_list_serializer = serializer.HostListSerializer(instance=instance)
return success(result=host_list_serializer.data)
@ -83,7 +85,9 @@ class HostModelViewSet(GenericViewSet,
instance = self.check_instance_exist(request, *args, **kwargs)
if not instance:
return not_found()
self.client_deploy_cmd_execute(instance, 'delete')
status, content = self.client_deploy_cmd_delete(instance)
if status != 200:
return other_response(message="删除失败，清除脚本执行失败，错误如下：{}".format(content.get("message")), code=400, success=False)
self.perform_destroy(instance)
return success(message="删除成功", code=200, result={})
@ -100,36 +104,24 @@ class HostModelViewSet(GenericViewSet,
instance = self.get_queryset().filter(**kwargs).first()
return instance if instance else None
def client_deploy_cmd_execute(self, instance, exec):
if exec == 'init':
thread = threading.Thread(target=self.client_deploy_cmd_init, args=(instance,))
thread.start()
if exec == 'delete':
thread = threading.Thread(target=self.client_deploy_cmd_delete, args=(instance,))
thread.start()
def client_deploy_cmd_init(self, instance):
url = settings.INIT_SERVER + "api/v1/tasks/"
data = {"service_name": "node_init",
"instance": instance.ip,
"update_host_status": True
}
headers = {
'Content-Type': "application/json"
}
data = json.dumps(data)
requests.post(url=url, data=data, headers=headers)
script_task(data)
def client_deploy_cmd_delete(self, instance):
url = settings.INIT_SERVER + "api/v1/tasks/"
data = {"service_name": "node_delete",
"instance": instance.ip
}
headers = {
'Content-Type': "application/json"
}
data = json.dumps(data)
requests.post(url=url, data=data, headers=headers)
try:
data = {"service_name": "node_delete",
"instance": instance.ip
}
resp = script_task(data)
logger.info(resp.status_code, resp.data)
return resp.status_code, resp.data
except Exception as e:
logger.error(e, exc_info=True)
return False, str(e)
class ClusterViewSet(GenericViewSet,

150
sysom_api/apps/task/views.py
View File

@ -54,44 +54,7 @@ class TaskAPIView(GenericViewSet,
def create(self, request, *args, **kwargs):
try:
data = request.data
params = data.copy()
service_name = data.pop("service_name", None)
update_host_status = data.pop("update_host_status", False)
task_id = uuid_8()
username = data['username'] if data.get('username') else "admin"
user = User.objects.filter(username=username).first()
if service_name:
SCRIPTS_DIR = settings.SCRIPTS_DIR
service_path = os.path.join(SCRIPTS_DIR, service_name)
if not os.path.exists(service_path):
return other_response(message="can not find script file, please check service name", code=400)
try:
resp = subprocess.run([service_path, json.dumps(data)], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as e:
JobModel.objects.create(command='', task_id=task_id,
created_by=user, result=str(e), status="Fail")
logger.error(e, exc_info=True)
return other_response(message=str(e), code=400, success=False)
if resp.returncode != 0:
JobModel.objects.create(command='', task_id=task_id,
created_by=user, result=resp.stderr.decode('utf-8'), status="Fail")
return other_response(message=str(resp.stderr.decode('utf-8')), code=400, success=False)
stdout = resp.stdout
stdout = stdout.decode('utf-8')
resp = ast.literal_eval(stdout)
resp_scripts = resp.get("commands")
if not resp_scripts:
JobModel.objects.create(command='', task_id=task_id,
created_by=user, result="not find commands, Please check the script return",
status="Fail")
return other_response(message="not find commands, Please check the script return", code=400)
self.ssh_job(resp_scripts, task_id, user, json.dumps(params), update_host_status=update_host_status,
service_name=service_name)
return success(result={"instance_id": task_id})
else:
return self.default_ssh_job(data, task_id)
return script_task(data)
except Exception as e:
logger.error(e, exc_info=True)
return other_response(message=str(e), code=400, success=False)
@ -103,36 +66,6 @@ class TaskAPIView(GenericViewSet,
response = seriaizer.JobRetrieveSerializer(instance)
return success(result=response.data)
def default_ssh_job(self, data, task_id):
try:
host_ids = data.get("host_ids")
commands = data.get("commands")
if not host_ids:
return other_response(message="请选择执行主机", code=400)
user = User.objects.filter(username='admin').first()
cmds = []
for i in range(len(host_ids)):
instance = {}
host = HostModel.objects.filter(pk=host_ids[i]).first()
instance["instance"] = host.ip
instance["cmd"] = commands[i]
cmds.append(instance)
self.ssh_job(cmds, task_id, user)
return success(result={"instance_id": task_id})
except Exception as e:
logger.error(e, exc_info=True)
return other_response(message=str(e), code=400, success=False)
def ssh_job(self, resp_scripts, task_id, user, data=None, **kwargs):
if not data:
job_model = JobModel.objects.create(command=resp_scripts, task_id=task_id,
created_by=user)
else:
job_model = JobModel.objects.create(command=resp_scripts, task_id=task_id,
created_by=user, params=data)
sch_job = SshJob(resp_scripts, job_model, **kwargs)
scheduler.add_job(sch_job.run)
def list(self, request, *args, **kwargs):
queryset = self.filter_queryset(self.get_queryset())
if not queryset:
@ -150,3 +83,84 @@ class TaskAPIView(GenericViewSet,
instance.deleted_at = human_datetime()
instance.deleted_by = self.request.user
instance.save()
def script_task(data):
try:
params = data.copy()
service_name = data.pop("service_name", None)
update_host_status = data.pop("update_host_status", False)
task_id = uuid_8()
username = data['username'] if data.get('username') else "admin"
user = User.objects.filter(username=username).first()
if service_name:
SCRIPTS_DIR = settings.SCRIPTS_DIR
service_path = os.path.join(SCRIPTS_DIR, service_name)
if not os.path.exists(service_path):
logger.error("can not find script file, please check service name")
return other_response(message="can not find script file, please check service name", code=400,
success=False)
try:
resp = subprocess.run([service_path, json.dumps(data)], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as e:
JobModel.objects.create(command='', task_id=task_id,
created_by=user, result=str(e), status="Fail")
logger.error(e, exc_info=True)
return other_response(message=str(e), code=400, success=False)
if resp.returncode != 0:
logger.error(str(resp.stderr.decode('utf-8')))
JobModel.objects.create(command='', task_id=task_id,
created_by=user, result=resp.stderr.decode('utf-8'), status="Fail")
return other_response(message=str(resp.stderr.decode('utf-8')), code=400, success=False)
stdout = resp.stdout
stdout = stdout.decode('utf-8')
resp = ast.literal_eval(stdout)
resp_scripts = resp.get("commands")
if not resp_scripts:
logger.error("not find commands, Please check the script return")
JobModel.objects.create(command='', task_id=task_id,
created_by=user, result="not find commands, Please check the script return",
status="Fail")
return other_response(message="not find commands, Please check the script return", code=400,
success=False)
ssh_job(resp_scripts, task_id, user, json.dumps(params), update_host_status=update_host_status,
service_name=service_name)
return success(result={"instance_id": task_id})
else:
return default_ssh_job(data, task_id)
except Exception as e:
logger.error(e, exc_info=True)
return other_response(message=str(e), code=400, success=False)
def default_ssh_job(data, task_id):
try:
host_ids = data.get("host_ids")
commands = data.get("commands")
if not host_ids:
return other_response(message="请选择执行主机", code=400)
user = User.objects.filter(username='admin').first()
cmds = []
for i in range(len(host_ids)):
instance = {}
host = HostModel.objects.filter(pk=host_ids[i]).first()
instance["instance"] = host.ip
instance["cmd"] = commands[i]
cmds.append(instance)
ssh_job(cmds, task_id, user)
return success(result={"instance_id": task_id})
except Exception as e:
logger.error(e, exc_info=True)
return other_response(message=str(e), code=400, success=False)
def ssh_job(resp_scripts, task_id, user, data=None, **kwargs):
if not data:
job_model = JobModel.objects.create(command=resp_scripts, task_id=task_id,
created_by=user)
else:
job_model = JobModel.objects.create(command=resp_scripts, task_id=task_id,
created_by=user, params=data)
sch_job = SshJob(resp_scripts, job_model, **kwargs)
scheduler.add_job(sch_job.run)

2
sysom_api/apps/vul/views.py
View File

@ -12,7 +12,7 @@ from apps.vul.vul import fix_cve, get_unfix_cve
logger = logging.getLogger(__name__)
scheduler = BackgroundScheduler(timezone=get_localzone())
scheduler = BackgroundScheduler(timezone=f'{get_localzone()}')
@register_job(scheduler, 'cron', id='update_vul', hour=0, minute=0)

1
sysom_api/conf/common.py
View File

@ -39,7 +39,6 @@ MIDDLEWARE = [
]
DEBUG = True
INIT_SERVER = 'http://127.0.0.1:8001/'
# Mysql数据库
DATABASES = {

39
sysom_api/lib/renderers.py
View File

@ -27,23 +27,22 @@ class SysomJsonRender(JSONRenderer):
method = request.method
result = response.data
if method != 'GET':
kwargs = {
'request_ip': request.META.get('REMOTE_ADDR', None),
'request_url': request.path,
'request_browser_agent': request.headers.get('User-Agent', ''),
'request_method': method,
'handler_view': view.__class__.__name__,
'response_status': result.get('code')
}
if 'auth' in request.path:
kwargs['request_option'] = 0
if result.get('code') == 200:
kwargs['user_id'] = result['data']['id']
else:
kwargs['request_option'] = 1
kwargs['user'] = user
try:
HandlerLog.objects.create(**kwargs)
except:
pass
kwargs = {
'request_ip': request.META.get('REMOTE_ADDR', None),
'request_url': request.path,
'request_browser_agent': request.headers.get('User-Agent', ''),
'request_method': method,
'handler_view': view.__class__.__name__,
'response_status': result.get('code')
}
if 'auth' in request.path:
kwargs['request_option'] = 0
if result.get('code') == 200:
kwargs['user_id'] = result['data']['id']
else:
kwargs['request_option'] = 1
kwargs['user'] = user
try:
HandlerLog.objects.create(**kwargs)
except:
pass

36
sysom_web/src/pages/journal/Audit/index.jsx
View File

@ -2,7 +2,14 @@ import { useRef } from 'react';
import { useIntl, FormattedMessage } from 'umi';
import { PageContainer } from '@ant-design/pro-layout';
import ProTable from '@ant-design/pro-table';
import { getAudit } from '../service';
import { getAudit, get_response_code } from '../service';
const request = async (params) => {
const response = await get_response_code(params)
return response.data;
};
const AuditList = () => {
const actionRef = useRef();
@ -13,10 +20,12 @@ const AuditList = () => {
title: <FormattedMessage id="pages.journal.audit.created_at" defaultMessage="created_at" />,
dataIndex: 'created_at',
valueType: 'dateTime',
hideInSearch: true
},
{
title: <FormattedMessage id="pages.journal.audit.username" defaultMessage="username" />,
dataIndex: 'username',
hideInSearch: true
},
{
title: <FormattedMessage id="pages.journal.audit.request_ip" defaultMessage="request_ip" />,
@ -32,11 +41,20 @@ const AuditList = () => {
title: <FormattedMessage id="pages.journal.audit.request_method" defaultMessage="request_method" />,
dataIndex: 'request_method',
valueType: 'textarea',
valueEnum: {
'get': { text: 'GET' },
'post': { text: 'POST' },
'put': { text: 'PUT' },
'delete': { text: 'DELETE' },
'patch': { text: 'PATCH' },
}
},
{
title: <FormattedMessage id="pages.journal.audit.response_status" defaultMessage="response_status" />,
dataIndex: 'response_status',
valueType: 'textarea',
valueType: 'select',
request,
params: {}
},
{
title: <FormattedMessage id="pages.journal.audit.request_option" defaultMessage="request_option" />,
@ -54,6 +72,20 @@ const AuditList = () => {
},
},
},
{
title: '日志时间',
dataIndex: 'created_at',
valueType: 'dateTimeRange',
hideInTable: true,
search: {
transform: (value) => {
return {
startTime: value[0],
endTime: value[1],
};
},
},
},
];
return (
<PageContainer>

17
sysom_web/src/pages/journal/service.js
View File

@ -16,6 +16,21 @@ export async function getAudit(params, options) {
params: params,
...(options || {}),
});
msg.data.reverse()
// msg.data.reverse()
return msg
}
export async function get_response_code(params, options) {
const token = localStorage.getItem('token');
const result = await request('/api/v1/response_code/', {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Authorization': token
},
params: params,
...(options || {}),
});
return result
}