Skip to content

Commit 40afb5a

Browse files
authored
Merge pull request #68 from fadetop/main
Add OMP-v1.1.0
2 parents 91bb113 + 14a020a commit 40afb5a

122 files changed

Lines changed: 3871 additions & 2943 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

UpdateLog.md

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,12 +109,20 @@
109109
- 支持通过前端界面方式卸载应用商店中已经发布的应用
110110
- 更新readme文档
111111

112-
## v1.0 (2023.07.14)
113-
- 新增功能「服务纳管」模块
114-
- 重构「服务自愈」模块
115-
- 新增OOM告警
116-
- 更新部分Grafana面板
112+
## v1.0.0 (2023.07.30)
113+
114+
- 新增功能【服务纳管】模块
115+
- 重构【服务自愈】模块
116+
- 新增 OOM 告警
117+
- 更新部分 Grafana 面板
117118
- 服务面板: redis、victoriametrics、rocketmq
118119
- 集群面板: redis、clickhouse、mysql、tengine
119120
- 修复 Grafana 无法登陆问题
120121

122+
## v1.1.0 (2023.09.30)
123+
124+
- 重构「数据备份」模块
125+
- 支持多端口服务监控 & 更新文档
126+
- 前端优化,步骤类交互型界面,刷新自动跳转
127+
- 前端优化,消除部分冗余导入
128+
- 修复bug:仪表盘异常清单类型缺失,环形统计图跳转增加类型过滤,nodeExporter、loki启动失败问题

config/omp.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ monitor_port:
9797
elasticsearchExporter: 19027
9898
httpdExporter: 19028
9999
igniteExporter: 19029
100+
rocketmqExporter: 19032
100101
# arangodb与nacos没有单独的exporter,使用的是arangodb和nacos的原生自带接口
101102
arangodbExporter: 18119
102103
nacosExporter: 18117
@@ -156,3 +157,6 @@ health_request_sleep: 6
156157
template_cluster_check: true
157158
#服务发现脚本,纳管自定义排查脚本
158159
service_discovery:
160+
backup_service:
161+
- mysql
162+
- postgreSql

omp_server/app_store/app_store_serializers.py

Lines changed: 85 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -686,7 +686,17 @@ def validate(self, attrs):
686686
pro_id_list = app_now.values_list("product_id", flat=True).distinct()
687687
# 验证 product 的依赖项均已包含
688688
pro_queryset = ProductHub.objects.filter(id__in=pro_id_list)
689+
app_target_all = ApplicationHub.objects.filter(
690+
product_id__in=pro_id_list)
691+
# 考虑到同产品下会有同名服务情况,做去重处理,按照时间版本号取最新
692+
app_target = []
689693
for pro in pro_queryset:
694+
for ser in json.loads(pro.pro_services):
695+
app_target.append(
696+
app_target_all.filter(
697+
app_name=ser["name"], app_version=ser["version"]
698+
).last()
699+
)
690700
# 无依赖项则跳过
691701
if not pro.pro_dependence:
692702
continue
@@ -711,26 +721,6 @@ def validate(self, attrs):
711721
app_target_all = ApplicationHub.objects.filter(
712722
product_id__in=pro_id_list)
713723

714-
# 考虑到同产品下会有同名服务情况,做去重处理,按照时间版本号取最新
715-
app_target_id_ls = []
716-
for pro_id in pro_id_list:
717-
# 同产品的 app_name 集合
718-
app_name_set = set()
719-
app_ls = app_target_all.filter(product_id=pro_id)
720-
for app in app_ls:
721-
if app.app_name not in app_name_set:
722-
new_version = app_ls.filter(
723-
app_name=app.app_name
724-
).order_by("-created").first().app_version
725-
if new_version == app.app_version:
726-
app_name_set.add(app.app_name)
727-
app_target_id_ls.append(app.id)
728-
729-
# 获取目标 app
730-
app_target = app_target_all.filter(
731-
id__in=app_target_id_ls, is_release=True
732-
)
733-
734724
# 所有 affinity 为 tengine 字段 (Web 服务),不参与比较
735725
now_set = set(filter(
736726
lambda x: x.extend_fields.get("affinity") != "tengine", app_now))
@@ -899,6 +889,81 @@ class Meta:
899889
"module", "module_id")
900890

901891

892+
class ProductCompositionSerializer(ModelSerializer):
893+
pro_ser_others = serializers.SerializerMethodField()
894+
pro_services = serializers.CharField(
895+
# child=serializers.DictField(),
896+
help_text="产品包含服务列表",
897+
required=True,
898+
error_messages={"required": "必须包含[pro_services]字段"}
899+
)
900+
pro_name = serializers.CharField(help_text="产品名称", required=True,
901+
error_messages={"required": "请填写产品名称"})
902+
pro_version = serializers.CharField(help_text="产品版本", required=True,
903+
error_messages={"required": "请填写产品版本"})
904+
905+
def get_pro_ser_others(self, obj, **kwargs):
906+
res_list = []
907+
all_apps_set, all_true_apps_set = set(), set()
908+
if obj.applicationhub_set.exists():
909+
all_apps = obj.applicationhub_set.values_list("app_name", "app_version")
910+
for app in all_apps:
911+
all_apps_set.add(",".join(app))
912+
pro_ser = kwargs.get('pro_ser')
913+
pro_services = pro_ser if pro_ser else json.loads(obj.pro_services)
914+
for t_app in pro_services:
915+
all_true_apps_set.add(f"{t_app['name']},{t_app['version']}")
916+
# 一部分用作校验用
917+
if pro_ser:
918+
return all_true_apps_set - all_apps_set
919+
920+
for r_app in all_apps_set - all_true_apps_set:
921+
r_app_ls = r_app.split(",")
922+
res_list.append(
923+
{
924+
"name": r_app_ls[0],
925+
"version": r_app_ls[1],
926+
}
927+
)
928+
return res_list
929+
930+
def validate_pro_services(self, pro_services):
931+
pro_services = json.loads(pro_services)
932+
if not isinstance(pro_services, list):
933+
raise ValidationError({
934+
"pro_services": "pro_services必须是list"
935+
})
936+
pro_ser_len = len(pro_services)
937+
ser_name = {}
938+
for app in pro_services:
939+
ser_name[app.get('name', "")] = app.get('version', '')
940+
if len(ser_name) != pro_ser_len:
941+
raise ValidationError({
942+
"pro_services": "产品内服务名称需保证唯一或字段传递异常"
943+
})
944+
return pro_services
945+
946+
def validate(self, attrs):
947+
pro_name = attrs.get("pro_name")
948+
pro_version = attrs.get("pro_version")
949+
pro_obj = ProductHub.objects.filter(pro_name=pro_name, pro_version=pro_version).first()
950+
if not pro_obj:
951+
raise ValidationError({
952+
"pro_services": "请填写正确的产品名称和版本"
953+
})
954+
955+
diff_ser = self.get_pro_ser_others(pro_obj, pro_ser=attrs.get("pro_services"))
956+
if diff_ser:
957+
raise ValidationError({
958+
"pro_services": f"存在不归属于当前产品的服务{diff_ser}"
959+
})
960+
return attrs
961+
962+
class Meta:
963+
model = ProductHub
964+
fields = ("pro_name", "pro_version", "pro_services", "pro_ser_others")
965+
966+
902967
class DeleteComponentSerializer(ModelSerializer):
903968
"""
904969
基础组件序列化

omp_server/app_store/tasks.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -782,13 +782,21 @@ def get_port(keyword):
782782
_ret_dic = {
783783
"listen_port": get_port("service_port"),
784784
"metric_port": None,
785+
"run_port": [],
785786
"only_process": False,
786787
"process_key_word": None,
787788
"type": None
788789
}
790+
run_port_key_list = list()
791+
run_port_value_list = list()
789792
app_monitor = detail_obj.service.service.app_monitor
790793
if not app_monitor:
791794
return False, _ret_dic
795+
run_port_key_list = app_monitor.get("run_port", [])
796+
if len(run_port_key_list) > 0:
797+
run_port_value_list = [get_port(key) for key in run_port_key_list]
798+
if None not in run_port_value_list:
799+
_ret_dic["run_port"] = run_port_value_list
792800
_ret_dic["type"] = app_monitor.get("type")
793801
_ret_dic["process_key_word"] = app_monitor.get("process_name")
794802

@@ -800,7 +808,6 @@ def get_port(keyword):
800808
else:
801809
_metric_port_key = None
802810
if detail_obj.service.service_port is not None:
803-
_ret_dic["listen_port"] = get_port("service_port")
804811
_ret_dic["metric_port"] = get_port(_metric_port_key)
805812
if _ret_dic["metric_port"]:
806813
return True, _ret_dic
@@ -837,6 +844,7 @@ def add_prometheus(main_history_id, queryset=None):
837844
# TODO 已是否具有端口作为是否需要添加监控的依据,后续版本优化
838845
instance_name = detail_obj.service.service_instance_name
839846
service_port = _monitor_dic.get("listen_port")
847+
run_port = _monitor_dic.get("run_port", [])
840848
# 获取数据目录、日志目录
841849
app_install_args = detail_obj.install_detail_args.get(
842850
"install_args", [])
@@ -864,6 +872,7 @@ def add_prometheus(main_history_id, queryset=None):
864872
"env": "default",
865873
"ip": detail_obj.service.ip,
866874
"listen_port": service_port,
875+
"run_port": run_port,
867876
"metric_port": _monitor_dic.get("metric_port"),
868877
"only_process": _monitor_dic.get("only_process"),
869878
"process_key_word": _monitor_dic.get("process_key_word"),

omp_server/app_store/urls.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717
ApplicationTemplateView, DeploymentPlanValidateView,
1818
DeploymentPlanImportView, DeploymentPlanListView,
1919
DeploymentOperableView, DeploymentTemplateView,
20-
ExecutionRecordAPIView, DeleteAppStorePackageView)
20+
ExecutionRecordAPIView, DeleteAppStorePackageView,
21+
ProductCompositionView)
2122
from app_store.views_for_install import (
2223
ComponentEntranceView,
2324
ProductEntranceView,
@@ -185,3 +186,10 @@
185186
ExecutionRecordAPIView,
186187
basename="ExecutionRecord"
187188
)
189+
190+
# 产品类型接口查看及修改
191+
router.register(
192+
"productComposition",
193+
ProductCompositionView,
194+
basename="productComposition"
195+
)

omp_server/app_store/views.py

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@
3737
PublishPackageHistorySerializer, DeploymentPlanValidateSerializer,
3838
DeploymentImportSerializer, DeploymentPlanListSerializer,
3939
ExecutionRecordSerializer, DeleteComponentSerializer,
40-
DeleteProDuctSerializer
40+
DeleteProDuctSerializer, ProductCompositionSerializer
4141
)
42-
from backups.backup_service import cmd
42+
from backups.backups_utils import cmd
4343
from omp_server.settings import PROJECT_DIR
4444

4545
from app_store.app_store_serializers import (
@@ -994,6 +994,41 @@ class ExecutionRecordAPIView(GenericViewSet, ListModelMixin):
994994
get_description = "查询执行记录"
995995

996996

997+
class ProductCompositionView(GenericViewSet, ListModelMixin,
998+
CreateModelMixin):
999+
serializer_class = ProductCompositionSerializer
1000+
# 关闭权限、认证设置
1001+
authentication_classes = ()
1002+
permission_classes = ()
1003+
1004+
get_description = "查询产品信息"
1005+
post_description = "修改产品包含服务信息"
1006+
1007+
def get_queryset(self):
1008+
return ProductHub.objects.filter(**self.request.query_params.dict())
1009+
1010+
def list(self, request, *args, **kwargs):
1011+
serializer = self.get_serializer(self.get_queryset(), many=True)
1012+
for data in serializer.data:
1013+
data["pro_services"] = json.loads(data.get("pro_services"))
1014+
return Response(serializer.data)
1015+
1016+
def create(self, request, *args, **kwargs):
1017+
pro_services = json.dumps(request.data.get("pro_services", []), ensure_ascii=False)
1018+
request.data["pro_services"] = pro_services
1019+
1020+
serializer = self.get_serializer(data=request.data)
1021+
serializer.is_valid(raise_exception=True)
1022+
params = request.data
1023+
pro_obj = ProductHub.objects.filter(
1024+
pro_name=params.get('pro_name'), pro_version=params.get('pro_version')
1025+
).first()
1026+
pro_obj.pro_services = pro_services
1027+
pro_obj.save()
1028+
1029+
return Response("修改成功")
1030+
1031+
9971032
class DeleteAppStorePackageView(GenericViewSet, ListModelMixin, CreateModelMixin):
9981033
"""
9991034
get:

omp_server/backups/__init__.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
11
# !/usr/bin/python3
22
# -*-coding:utf-8-*-
3-
# Author: lingyang guo

omp_server/backups/admin.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
11
# !/usr/bin/python3
22
# -*-coding:utf-8-*-
3-
# Author: lingyang guo

omp_server/backups/apps.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
# !/usr/bin/python3
22
# -*-coding:utf-8-*-
3-
# Author: lingyang guo
43

54
from django.apps import AppConfig
65

0 commit comments

Comments
 (0)