fix(alarm): Optimize count query speed

environment/1_sdk/sysom_utils/fastapi_helper.py

@@ -13,6 +13,8 @@ from pydantic import BaseModel, create_model
 from starlette.requests import Request
 from sqlalchemy import desc, asc, and_
 from sqlalchemy.orm import Session, Query
+from sqlalchemy.sql import func
+from sqlalchemy.orm.attributes import InstrumentedAttribute
 from sqlalchemy.sql.sqltypes import Integer, String, Enum, SmallInteger
 
 
@@ -20,6 +22,7 @@ from sqlalchemy.sql.sqltypes import Integer, String, Enum, SmallInteger
 # CRUD Helper
 #######################################################################################
 
+
 class BaseQueryParamsInterface:
     @abstractmethod
     def get_skip(self) -> Optional[int]:
@@ -41,6 +44,7 @@ class BaseQueryParamsInterface:
     def get_sorter(self) -> Any:
         pass
 
+
 class QueryBuilder:
     def __init__(self, queryset: Query, query_params: BaseQueryParamsInterface) -> None:
         self.queryset: Query = queryset
@@ -152,6 +156,24 @@ class BaseQueryParams(BaseModel):
     def get_query_builder(self, db: Session) -> QueryBuilder:
         return QueryBuilder(db.query(self.get_model_class()), self)
 
+    def get_count_by(self, db: Session, attr: InstrumentedAttribute) -> int:
+        """Get the total number of entries
+
+        Args:
+            db (Session): _description_
+            attr (InstrumentedAttribute): _description_
+
+        Returns:
+            int: total number of entries
+        Reference:
+            https://blog.csdn.net/chenhepg/article/details/105169255
+        """
+        exp = db.query(func.count(attr))
+        filters = self.get_filters()
+        if filters is not None:
+            exp = exp.filter(filters)
+        return exp.scalar()
+
     def get_query_exp(self, db: Session) -> Any:
         exp = db.query(self.get_model_class())
         filters = self.get_filters()
@@ -349,7 +371,9 @@ class StandardListResponse(Generic[M, S]):
             raise Exception("data must be orm model object or pydantic object")
         if total == 0:
             total = len(response_data)
-        return response_type(code=code, message=message, data=response_data, total=total)
+        return response_type(
+            code=code, message=message, data=response_data, total=total
+        )
 
     @classmethod
     def success(

sysom_server/sysom_alarm/app/crud.py

@@ -8,7 +8,7 @@ Description:
 """
 from typing import List, Tuple
 from clogger import logger
-from sqlalchemy.orm import Session, lazyload, joinedload
+from sqlalchemy.orm import Session, joinedload
 from app import models, schemas, query
 from sysom_utils import dict_merge
 
@@ -98,18 +98,16 @@ def get_alert_datas(
 
     Args:
         db (Session): _description_
-        skip (int, optional): _description_. Defaults to 0.
-        limit (int, optional): _description_. Defaults to 10.
+        query_params (query.AlertDataQueryParams): _description_
     """
-    query_builder = query_params.get_query_builder(db)
-
     # 1. Get total count after apply filter
-    query_builder.apply_filter()
-    total_count = len(query_builder.build().all())
+    total_count = query_params.get_count_by(db, models.AlertData.id)
 
     # 2. Get alert list after apply sorter and paging
     alert_list = (
-        query_builder.apply_sorter()
+        query_params.get_query_builder(db)
+        .apply_filter()
+        .apply_sorter()
         .apply_paging()
         .build()
         .options(joinedload(models.AlertData.merged_alerts))

sysom_server/sysom_alarm/app/executor.py

@@ -83,7 +83,7 @@ class AlarmListener(MultiConsumer):
                 alert_id = action_data["alert_id"]
                 opt = action_data["opt"]
                 opt_key = opt.pop("key")
-                extra_annotations = {opt_key: opt}
+                extra_annotations = {f"SYSOM_ALARM:OPT:{opt_key}": opt}
                 append_alert_annotations(db, alert_id, extra_annotations)
             elif action == "MERGE":
                 # Perform merge

sysom_server/sysom_alarm/app/query.py

@@ -19,6 +19,7 @@ class AlertDataQueryParams(BaseQueryParams):
     sort__: str = "-alert_time"
 
     # 3. 定义支持用于过滤的参数
+    alert_id: Optional[str] = None
     instance: Optional[str] = None
     alert_item: Optional[str] = None
     alert_category: Optional[str] = None

sysom_server/sysom_alarm/app/routers/alert.py

@@ -6,7 +6,6 @@ Email               mfeng@linux.alibaba.com
 File                health.py
 Description:
 """
-from typing import List, Generic
 from clogger import logger
 from fastapi import APIRouter, Depends
 from sqlalchemy.orm import Session