commit 1bb6f0b9a812fc2574789958105f47c2b7ac7866 Author: hukdoesn Date: Thu Jun 12 16:48:37 2025 +0800 first commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2a44f12 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,70 @@ +# 通用忽略 +.git +.gitignore +.DS_Store +.env +.env.* +.vscode +.idea +*.swp +*.swo +README.md +LICENSE + +# 前端忽略 +web/node_modules +web/.vscode +web/.idea +web/coverage +web/npm-debug.log +web/yarn-debug.log +web/yarn-error.log +web/pnpm-debug.log +web/.env.local +web/.env.development.local +web/.env.test.local +web/.env.production.local +# 注意：不要忽略web/dist目录，因为我们需要将其复制到容器中 + +# 后端忽略 +backend/__pycache__/ +backend/**/__pycache__/ +backend/**/**/__pycache__/ +backend/*.py[cod] +backend/*$py.class +backend/*.so +backend/.Python +backend/env/ +backend/build/ +backend/develop-eggs/ +backend/dist/ +backend/downloads/ +backend/eggs/ +backend/.eggs/ +backend/lib/ +backend/lib64/ +backend/parts/ +backend/sdist/ +backend/var/ +backend/*.egg-info/ +backend/.installed.cfg +backend/*.egg +backend/db.sqlite3 +backend/media +backend/logs/ +backend/backups/ + +# 过滤掉migrations文件，但保留__init__.py +backend/*/migrations/* +!backend/*/migrations/__init__.py + +# 其他文件 +Dockerfile +docker-compose.yml +.dockerignore +start-containers.sh +verify-docker.sh +DOCKER_IN_DOCKER.md +# 不要忽略启动脚本 +!docker-entrypoint.sh +!ci-entrypoint-dind.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d2f0fb6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,43 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +*.egg + +# Django +*.log +*.pot +*.pyc +/backend/*/migrations/ +**/migrations/ +db.sqlite3 +/backend/backups/ +/liteops-www/ +package.json + +# Node +node_modules/ +package-lock.json +**/node_modules/ +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? +*.gz +*.xml + +# web +/web/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7da8ea6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,252 @@ +# ============================================================================= +# LiteOps CI/CD Platform - Docker in Docker Multi-stage Dockerfile +# ============================================================================= +# 第一阶段：构建和工具安装阶段 +FROM debian:bullseye-slim AS builder + +# 设置构建时的环境变量 +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + # Java环境变量 + JAVA_HOME=/usr/local/java/jdk1.8.0_211 \ + MAVEN_HOME=/usr/local/maven/apache-maven-3.8.8 \ + # NVM环境变量 + NVM_DIR=/root/.nvm \ + # Docker版本 + DOCKER_VERSION=24.0.7 + +# ============================================================================= +# 系统基础配置和轻量化软件安装 +# ============================================================================= +RUN set -eux; \ + # 配置阿里云镜像源以加速下载 + sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list && \ + sed -i 's/security.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + # Python + python3.9 \ + python3-pip \ + curl \ + ca-certificates \ + # SSH + openssh-client \ + # Git（GitPython依赖） + git \ + # 进程管理 + procps \ + bash \ + # Docker安装依赖 + apt-transport-https \ + gnupg \ + lsb-release \ + iptables \ + && \ + # 创建Python符号链接 + ln -sf /usr/bin/python3.9 /usr/bin/python3 && \ + ln -sf /usr/bin/python3.9 /usr/bin/python && \ + # 配置pip镜像源 + pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ && \ + pip config set install.trusted-host mirrors.aliyun.com && \ + # SSH客户端基础配置 + mkdir -p /root/.ssh && \ + chmod 700 /root/.ssh && \ + # 轻量化安装NVM + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash && \ + echo 'export NVM_DIR="$HOME/.nvm"' >> /root/.bashrc && \ + echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> /root/.bashrc && \ + echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" --no-use' >> /root/.profile && \ + # 创建Java和Maven安装目录 + mkdir -p /usr/local/java /usr/local/maven && \ + # 安装精简版Docker Engine + (curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://mirrors.aliyun.com/docker-ce/linux/debian bullseye stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null) || \ + (curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian bullseye stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null) && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + docker-ce-cli \ + docker-ce \ + && \ + apt-get autoremove -y && \ + apt-get autoclean && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/cache/apt/* /root/.cache/* + +# ============================================================================= +# 精简Java环境安装 +# ============================================================================= +COPY jdk-8u211-linux-x64.tar.gz apache-maven-3.8.8-bin.tar.gz /tmp/ + +RUN set -eux; \ + # 解压JDK和Maven + tar -xzf /tmp/jdk-8u211-linux-x64.tar.gz -C /usr/local/java && \ + tar -xzf /tmp/apache-maven-3.8.8-bin.tar.gz -C /usr/local/maven && \ + # 立即清理压缩包 + rm -f /tmp/jdk-8u211-linux-x64.tar.gz /tmp/apache-maven-3.8.8-bin.tar.gz && \ + # 极度精简JDK - 删除所有不必要的文件 + cd /usr/local/java/jdk1.8.0_211 && \ + rm -rf src.zip javafx-src.zip man sample demo \ + COPYRIGHT LICENSE README.html THIRDPARTYLICENSEREADME.txt \ + release ASSEMBLY_EXCEPTION && \ + # 删除不常用的JDK工具（保留核心编译和运行工具） + cd bin && \ + rm -f appletviewer extcheck jarsigner java-rmi.cgi \ + javadoc javah javap javaws jcmd jconsole jdb jhat \ + jinfo jmap jps jrunscript jsadebugd jstack jstat \ + jstatd jvisualvm native2ascii orbd policytool \ + rmic rmid rmiregistry schemagen serialver servertool \ + tnameserv wsgen wsimport xjc && \ + # 删除JRE中的不必要文件 + cd ../jre && \ + rm -rf COPYRIGHT LICENSE README THIRDPARTYLICENSEREADME.txt \ + ASSEMBLY_EXCEPTION release && \ + cd bin && \ + rm -f javaws jvisualvm orbd policytool rmid \ + rmiregistry servertool tnameserv && \ + # 精简Maven安装，删除文档和示例 + cd /usr/local/maven/apache-maven-3.8.8 && \ + rm -rf LICENSE NOTICE README.txt + +# ============================================================================= +# 第二阶段：超轻量运行时镜像 +# ============================================================================= +FROM debian:bullseye-slim + +# 设置运行时环境变量 +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + # Java环境变量 + JAVA_HOME=/usr/local/java/jdk1.8.0_211 \ + MAVEN_HOME=/usr/local/maven/apache-maven-3.8.8 \ + # NVM环境变量 + NVM_DIR=/root/.nvm \ + # Docker版本 + DOCKER_VERSION=24.0.7 \ + # Locale配置 - 使用POSIX避免SSH locale警告 + LC_ALL=POSIX \ + LANG=POSIX \ + # 更新PATH环境变量 + PATH=/usr/local/java/jdk1.8.0_211/bin:/usr/local/maven/apache-maven-3.8.8/bin:/usr/local/bin:/usr/local/sbin:$PATH + +# ============================================================================= +# 运行时最小化系统配置 +# ============================================================================= +RUN set -eux; \ + # 配置阿里云镜像源 + sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list && \ + sed -i 's/security.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list && \ + # 只安装绝对必需的运行时包 + apt-get update && \ + apt-get install -y --no-install-recommends \ + python3.9 \ + python3-pip \ + curl \ + ca-certificates \ + # SSH + openssh-client \ + # Git（GitPython依赖） + git \ + # 轻量web服务器 + nginx-light \ + # 进程管理 + procps \ + bash \ + # Docker运行时依赖 + apt-transport-https \ + gnupg \ + lsb-release \ + iptables \ + && \ + # 创建Python符号链接 + ln -sf /usr/bin/python3.9 /usr/bin/python3 && \ + ln -sf /usr/bin/python3.9 /usr/bin/python && \ + # 配置pip镜像源 + pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ && \ + pip config set install.trusted-host mirrors.aliyun.com && \ + # 安装精简版Docker Engine + (curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://mirrors.aliyun.com/docker-ce/linux/debian bullseye stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null) || \ + (curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian bullseye stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null) && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + docker-ce-cli \ + docker-ce \ + && \ + # 安装kubectl - 使用官方二进制文件 + KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) && \ + curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \ + chmod +x kubectl && \ + mv kubectl /usr/local/bin/ && \ + # 创建必要的目录 + mkdir -p /app/logs && \ + rm -rf /var/log/nginx/* /var/lib/nginx/body /var/lib/nginx/fastcgi \ + /var/lib/nginx/proxy /var/lib/nginx/scgi /var/lib/nginx/uwsgi \ + /etc/nginx/sites-enabled/default && \ + apt-get autoremove -y && \ + apt-get autoclean && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/cache/apt/* /root/.cache/* \ + /var/cache/debconf/* /var/lib/dpkg/info/* /usr/share/doc/* \ + /usr/share/man/* /usr/share/locale/* /usr/share/info/* + +# ============================================================================= +# 从构建阶段复制精简的文件 +# ============================================================================= +# 复制SSH配置 +COPY --from=builder /root/.ssh /root/.ssh + +# 复制精简的NVM环境 +COPY --from=builder /root/.nvm /root/.nvm +COPY --from=builder /root/.bashrc /root/.bashrc +COPY --from=builder /root/.profile /root/.profile + +# 复制精简后的Java环境 +COPY --from=builder /usr/local/java /usr/local/java +COPY --from=builder /usr/local/maven /usr/local/maven + +# Docker已在运行时阶段安装，无需复制 + +# ============================================================================= +# 应用程序配置 +# ============================================================================= +# 设置工作目录 +WORKDIR /app + +# 配置Nginx - 复制自定义配置文件 +COPY web/nginx.conf /etc/nginx/sites-available/default +RUN ln -sf /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default + +# 复制前端构建文件到Nginx静态文件目录 +COPY web/dist/ /usr/share/nginx/html/ + +# 优化Python依赖安装 +COPY backend/requirements.txt /app/ +RUN pip install --no-cache-dir -r requirements.txt && \ + # 清理pip缓存和不必要的文件 + rm -rf /root/.cache/pip /tmp/* && \ + # 移除pip的缓存目录 + pip cache purge 2>/dev/null || true + +# 复制后端应用代码 +COPY backend/ /app/ + +# 复制启动脚本并设置执行权限 +COPY docker-entrypoint.sh /app/ +COPY ci-entrypoint-dind.sh /usr/local/bin/ +RUN chmod +x /app/docker-entrypoint.sh /usr/local/bin/ci-entrypoint-dind.sh + +# ============================================================================= +# 容器配置 +# ============================================================================= +# 暴露端口 +# 80: Nginx Web服务器端口 +# 8900: Django后端API端口 +EXPOSE 80 8900 + +# 设置容器入口点和默认命令 +ENTRYPOINT ["/usr/local/bin/ci-entrypoint-dind.sh"] +CMD ["/app/docker-entrypoint.sh"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e698640 --- /dev/null +++ b/README.md @@ -0,0 +1,113 @@ +
+ +# 🚀 LiteOps - 轻量级DevOps平台 + +LiteOps Logo + +**简单、高效的CI/CD解决方案** + +
+ +

+ Vue 3 + Django + MySQL + Docker +

+ +# 项目介绍 + +## LiteOps CICD 平台概述 + +LiteOps是一个实用型的CI/CD平台。它并非追求大而全的DevOps解决方案，而是聚焦于团队日常工作中真正需要的自动化构建、和部署功能，帮助开发团队提高效率，减少重复性工作。 + +## 项目特点 + +LiteOps的核心特点是"实用、贴合需求、易于使用"： + +- **实用为先**：基于公司现有流程开发，解决实际问题，没有多余花里胡哨功能 +- **贴合需求**：针对团队缺少的功能进行定制开发，填补工作流程中的空白 +- **易于使用**：简洁直观的界面设计，降低使用门槛，减少学习成本，倾向于Jenkins 自由风格Job + +## 项目背景 + +在日常开发工作中，我发现现有的工作流程存在一些功能缺失。市面上的CI/CD工具虽然功能丰富，但往往存在以下问题： + +1. 与公司现有流程不匹配，需要大量定制。比如我们不允许自动化构建（webhook调用），只允许测试手动构建，便于知道发布之后修改了什么功能/bug。 +2. 功能过于复杂，团队实际只需要其中一小部分 +3. 学习和维护成本高（Jenkins Pipeline） +4. 难以满足团队特定的自动化需求 + +LiteOps正是基于这些实际问题开发的，它不追求"高大上"的全面解决方案，而是专注于解决团队日常工作中的实际痛点，提供刚好满足需求的功能。更多的是发布记录功能。如：测试去构建的时候需要去填写构建需求、可观测发布分支最后提交人以及提交commit记录。 + +## 技术架构 + +LiteOps采用前后端分离的架构设计： + +### 前端技术栈 + +- **Vue 3**：渐进式JavaScript框架 +- **Ant Design Vue 4.x**：基于Vue的UI组件库 +- **Axios**：基于Promise的HTTP客户端 +- **Vue Router**：Vue官方路由管理器 +- **AntV G2**：数据可视化图表库 + +### 后端技术栈 + +- **Django 4.2**：Python Web框架 +- **Django Channels**：WebSocket支持 +- **MySQL 8**：关系型数据库 +- **GitPython**：Git操作库 +- **Python-GitLab**：GitLab API客户端 +- **JWT认证**：用户身份验证 + +### 部署方案 + +- **Docker**：容器化部署 + +## 项目目标 + +LiteOps的目标是解决团队在开发流程中的实际问题，具体包括： + +1. 自动化团队中重复性高的构建和部署工作，节省人力成本 +2. 标准化项目的构建流程，减少人为错误 +3. 提供清晰的构建状态和日志，方便问题排查 +4. 支持团队特有的部署需求，适应现有的服务器环境 +5. 简化权限管理 + +## 适用场景 + +LiteOps主要适用于以下场景： + +- 需要解决特定CI/CD痛点的开发团队 +- 现有流程中缺少自动化构建和部署环节的项目 +- 希望减少手动操作、提高效率的开发环境 +- 对现有工具不满意，需要更贴合实际工作流程的解决方案 + +## 项目当前状态与未来规划 + +LiteOps目前处于未完善状态，虽然核心功能已经初步实现，但仍有许多需求和功能有待完善，如实现部署k8s项目。我希望通过开放的方式收集更多的需求和建议，使这个项目能够更好地服务于实际开发场景。 + +### 需求征集 + +我诚挚邀请你在查看[功能介绍文档](https://liteops.ext4.cn)和了解LiteOps后，提供宝贵的意见和建议： + +功能介绍文档：https://liteops.ext4.cn + +- **功能需求**：你希望看到哪些新功能或改进？ +- **用户体验**：界面和操作流程是否符合你的使用习惯？ +- **实际场景**：在你的工作环境中，有哪些CI/CD痛点尚未解决？ + +### 开源计划 + +在收集并实现足够的功能需求后，能够简单的支持一些团队正常使用，我计划将LiteOps完全开源，让更多的团队能够受益于这个项目。你的每一条建议都将帮助我打造一个更实用、更贴合实际需求的CI/CD工具。 + + +## 📞 联系我 + +如果您对LiteOps有任何建议、问题或需求，欢迎通过以下方式联系我们： + +- **邮箱**：hukdoesn@163.com +- **GitHub Issues**：[提交问题或建议](https://github.com/hukdoesn/liteops/issues) + +--- \ No newline at end of file diff --git a/backend/apps/__init__.py b/backend/apps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/apps/admin.py b/backend/apps/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/backend/apps/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/backend/apps/apps.py b/backend/apps/apps.py new file mode 100644 index 0000000..03f7bcd --- /dev/null +++ b/backend/apps/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class AppsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps' diff --git a/backend/apps/management/__init__.py b/backend/apps/management/__init__.py new file mode 100644 index 0000000..0519ecb --- /dev/null +++ b/backend/apps/management/__init__.py @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/backend/apps/management/commands/__init__.py b/backend/apps/management/commands/__init__.py new file mode 100644 index 0000000..0519ecb --- /dev/null +++ b/backend/apps/management/commands/__init__.py @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/backend/apps/models.py b/backend/apps/models.py new file mode 100644 index 0000000..ece8c15 --- /dev/null +++ b/backend/apps/models.py @@ -0,0 +1,410 @@ +from django.db import models + +class User(models.Model): + """ + 用户表 + """ + id = models.AutoField(primary_key=True) + user_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='用户ID') + username = models.CharField(max_length=50, unique=True, null=True, verbose_name='用户名') + name = models.CharField(max_length=50, null=True, verbose_name='姓名') + password = models.CharField(max_length=128, null=True, verbose_name='密码') + email = models.EmailField(max_length=100, unique=True, null=True, verbose_name='邮箱') + status = models.SmallIntegerField(null=True, verbose_name='状态') + login_time = models.DateTimeField(null=True, verbose_name='最后登录时间') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'user' + verbose_name = '用户' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return self.username + + +class Role(models.Model): + """ + 角色表 + """ + id = models.AutoField(primary_key=True) + role_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='角色ID') + name = models.CharField(max_length=50, unique=True, null=True, verbose_name='角色名称') + description = models.TextField(null=True, blank=True, verbose_name='角色描述') + permissions = models.JSONField(default=dict, null=True, verbose_name='权限配置') + creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'role' + verbose_name = '角色' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return self.name + + +class UserRole(models.Model): + """ + 用户角色关联表 + """ + id = models.AutoField(primary_key=True) + user = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='用户') + role = models.ForeignKey('Role', on_delete=models.CASCADE, to_field='role_id', null=True, verbose_name='角色') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'user_role' + verbose_name = '用户角色关联' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + unique_together = ['user', 'role'] # 确保用户和角色的组合唯一 + + def __str__(self): + return f"{self.user.username} - {self.role.name}" + + +class UserToken(models.Model): + """ + 用户Token表 + """ + id = models.AutoField(primary_key=True) + token_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='TokenID') + user = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='用户') + token = models.CharField(max_length=256, null=True, verbose_name='Token信息') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'user_token' + verbose_name = '用户Token' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return f"{self.user.username}'s token" + + +class Project(models.Model): + """ + 项目表 - 包含项目基本信息和服务信息 + """ + id = models.AutoField(primary_key=True) + project_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='项目ID') + name = models.CharField(max_length=50, null=True, verbose_name='项目名称') + description = models.TextField(null=True, blank=True, verbose_name='项目描述') + category = models.CharField(max_length=20, null=True, verbose_name='服务类别') # frontend, backend, mobile + repository = models.CharField(max_length=255, null=True, verbose_name='GitLab仓库地址') + creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'project' + verbose_name = '项目' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return self.name + + +class GitlabTokenCredential(models.Model): + """ + GitLab Token凭证表 + """ + id = models.AutoField(primary_key=True) + credential_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='凭证ID') + name = models.CharField(max_length=50, null=True, verbose_name='凭证名称') + description = models.TextField(null=True, blank=True, verbose_name='凭证描述') + token = models.CharField(max_length=255, null=True, verbose_name='GitLab Token') + creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'gitlab_token_credential' + verbose_name = 'GitLab Token凭证' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return self.name + + +class SSHKeyCredential(models.Model): + """ + SSH密钥凭证表 + """ + id = models.AutoField(primary_key=True) + credential_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='凭证ID') + name = models.CharField(max_length=50, null=True, verbose_name='凭证名称') + description = models.TextField(null=True, blank=True, verbose_name='凭证描述') + private_key = models.TextField(null=True, verbose_name='SSH私钥内容') + passphrase = models.CharField(max_length=255, null=True, blank=True, verbose_name='私钥密码 (可选)') + creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'ssh_key_credential' + verbose_name = 'SSH密钥凭证' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return self.name + + +class KubeconfigCredential(models.Model): + """ + Kubeconfig访问凭证表 + """ + id = models.AutoField(primary_key=True) + credential_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='凭证ID') + name = models.CharField(max_length=50, null=True, verbose_name='凭证名称') + description = models.TextField(null=True, blank=True, verbose_name='凭证描述') + kubeconfig_content = models.TextField(null=True, verbose_name='Kubeconfig文件内容') + cluster_name = models.CharField(max_length=100, null=True, verbose_name='集群名称') # 用于显示和区分 + context_name = models.CharField(max_length=100, null=True, verbose_name='上下文名称') # kubectl使用的context名称 + creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'kubeconfig_credential' + verbose_name = 'Kubeconfig访问凭证' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return self.name + + +class Environment(models.Model): + """ + 环境配置表 - 用于管理不同的部署环境（如开发、测试、预发布、生产等） + """ + id = models.AutoField(primary_key=True) + environment_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='环境ID') + name = models.CharField(max_length=50, null=True, verbose_name='环境名称') + type = models.CharField(max_length=20, null=True, verbose_name='环境类型') # development, testing, staging, production + description = models.TextField(null=True, blank=True, verbose_name='环境描述') + creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'environment' + verbose_name = '环境配置' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return self.name + + +class BuildTask(models.Model): + """构建任务表""" + id = models.AutoField(primary_key=True) + task_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='任务ID') + name = models.CharField(max_length=100, null=True, verbose_name='任务名称') + project = models.ForeignKey('Project', on_delete=models.CASCADE, to_field='project_id', null=True, verbose_name='所属项目') + environment = models.ForeignKey('Environment', on_delete=models.CASCADE, to_field='environment_id', null=True, verbose_name='构建环境') + description = models.TextField(null=True, blank=True, verbose_name='任务描述') + requirement = models.TextField(null=True, blank=True, verbose_name='构建需求描述') + branch = models.CharField(max_length=100, default='main', null=True, verbose_name='默认分支') + git_token = models.ForeignKey('GitlabTokenCredential', on_delete=models.SET_NULL, to_field='credential_id', null=True, verbose_name='Git Token') + version = models.CharField(max_length=50, null=True, blank=True, verbose_name='构建版本号') + + # 构建阶段 + stages = models.JSONField(default=list, verbose_name='构建阶段') + + # 外部脚本库配置 + use_external_script = models.BooleanField(default=False, verbose_name='使用外部脚本库') + external_script_config = models.JSONField(default=dict, verbose_name='外部脚本库配置', help_text=''' + { + "repo_url": "https://github.com/example/scripts.git", # Git仓库地址 + "directory": "/data/scripts", # 存放目录 + "branch": "main", # 分支名称（可选） + "token_id": "credential_id" # Git Token凭证ID（私有仓库） + } + ''') + + # 构建时间信息（使用JSON存储） + build_time = models.JSONField(default=dict, verbose_name='构建时间信息', help_text=''' + { + "total_duration": "300", # 总耗时（秒） + "start_time": "2024-03-06 12:00:00", # 开始时间 + "end_time": "2024-03-06 12:05:00", # 结束时间 + "stages_time": [ # 各阶段时间信息 + { + "name": "代码拉取", + "start_time": "2024-03-06 12:00:00", + "duration": "60" # 耗时（秒） + } + ] + } + ''') + + # 构建后操作 + notification_channels = models.JSONField(default=list, verbose_name='通知方式') + + # 状态和统计 + status = models.CharField(max_length=20, default='created', null=True, verbose_name='任务状态') # created, disabled + building_status = models.CharField(max_length=20, default='idle', null=True, verbose_name='构建状态') # idle, building + last_build_number = models.IntegerField(default=0, verbose_name='最后构建号') + total_builds = models.IntegerField(default=0, verbose_name='总构建次数') + success_builds = models.IntegerField(default=0, verbose_name='成功构建次数') + failure_builds = models.IntegerField(default=0, verbose_name='失败构建次数') + + creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'build_task' + verbose_name = '构建任务' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + +class BuildHistory(models.Model): + """构建历史表""" + id = models.AutoField(primary_key=True) + history_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='历史ID') + task = models.ForeignKey('BuildTask', on_delete=models.CASCADE, to_field='task_id', null=True, verbose_name='构建任务') + build_number = models.IntegerField(verbose_name='构建序号') + branch = models.CharField(max_length=100, null=True, verbose_name='构建分支') + commit_id = models.CharField(max_length=40, null=True, verbose_name='Git Commit ID') + version = models.CharField(max_length=50, null=True, verbose_name='构建版本') + status = models.CharField(max_length=20, default='pending', verbose_name='构建状态') # pending, running, success, failed, terminated + requirement = models.TextField(null=True, blank=True, verbose_name='构建需求描述') + build_log = models.TextField(null=True, blank=True, verbose_name='构建日志') + stages = models.JSONField(default=list, verbose_name='构建阶段') + build_time = models.JSONField(default=dict, verbose_name='构建时间信息', help_text=''' + { + "total_duration": "300", # 总耗时（秒） + "start_time": "2024-03-06 12:00:00", # 开始时间 + "end_time": "2024-03-06 12:05:00", # 结束时间 + "stages_time": [ # 各阶段时间信息 + { + "name": "代码拉取", + "start_time": "2024-03-06 12:00:00", + "duration": "60" # 耗时（秒） + } + ] + } + ''') + + operator = models.ForeignKey('User', on_delete=models.SET_NULL, to_field='user_id', null=True, verbose_name='构建人') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'build_history' + verbose_name = '构建历史' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + unique_together = ['task', 'build_number'] # 确保任务和构建号的组合唯一 + + def __str__(self): + return f"{self.task.name} #{self.build_number}" + + +class NotificationRobot(models.Model): + """通知机器人表""" + id = models.AutoField(primary_key=True) + robot_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='机器人ID') + type = models.CharField(max_length=20, null=True, verbose_name='机器人类型') # dingtalk, wecom, feishu + name = models.CharField(max_length=50, null=True, verbose_name='机器人名称') + webhook = models.CharField(max_length=255, null=True, verbose_name='Webhook地址') + security_type = models.CharField(max_length=20, null=True, verbose_name='安全设置类型') # none, secret, keyword, ip + secret = models.CharField(max_length=255, null=True, blank=True, verbose_name='加签密钥') + keywords = models.JSONField(default=list, null=True, blank=True, verbose_name='自定义关键词') + ip_list = models.JSONField(default=list, null=True, blank=True, verbose_name='IP白名单') + remark = models.TextField(null=True, blank=True, verbose_name='备注') + creator = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='创建者') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'notification_robot' + verbose_name = '通知机器人' + verbose_name_plural = verbose_name + ordering = ['-create_time'] + + def __str__(self): + return f"{self.name} ({self.type})" + + +class LoginLog(models.Model): + """ + 登录日志表 + """ + id = models.AutoField(primary_key=True) + log_id = models.CharField(max_length=32, unique=True, null=True, verbose_name='日志ID') + user = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='用户') + ip_address = models.CharField(max_length=50, null=True, verbose_name='IP地址') + user_agent = models.TextField(null=True, blank=True, verbose_name='用户代理') + status = models.CharField(max_length=20, null=True, verbose_name='登录状态') # success, failed + fail_reason = models.CharField(max_length=100, null=True, blank=True, verbose_name='失败原因') + login_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='登录时间') + + class Meta: + db_table = 'login_log' + verbose_name = '登录日志' + verbose_name_plural = verbose_name + ordering = ['-login_time'] + + def __str__(self): + return f"{self.user.username} - {self.login_time}" + + +class SecurityConfig(models.Model): + """ + 安全配置表 - 存储系统安全策略配置 + """ + id = models.AutoField(primary_key=True) + min_password_length = models.IntegerField(default=8, verbose_name='密码最小长度') + password_complexity = models.JSONField(default=list, verbose_name='密码复杂度要求', help_text='包含: uppercase, lowercase, number, special') + session_timeout = models.IntegerField(default=120, verbose_name='会话超时时间(分钟)') + max_login_attempts = models.IntegerField(default=5, verbose_name='最大登录尝试次数') + lockout_duration = models.IntegerField(default=30, verbose_name='账户锁定时间(分钟)') + enable_2fa = models.BooleanField(default=False, verbose_name='启用双因子认证') + update_time = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间') + + class Meta: + db_table = 'security_config' + verbose_name = '安全配置' + verbose_name_plural = verbose_name + + def __str__(self): + return "系统安全配置" + + +class LoginAttempt(models.Model): + """ + 登录尝试记录表 - 用于跟踪用户登录失败次数和账户锁定状态 + """ + id = models.AutoField(primary_key=True) + user = models.ForeignKey('User', on_delete=models.CASCADE, to_field='user_id', null=True, verbose_name='用户') + ip_address = models.CharField(max_length=50, null=True, verbose_name='IP地址') + failed_attempts = models.IntegerField(default=0, verbose_name='失败尝试次数') + locked_until = models.DateTimeField(null=True, blank=True, verbose_name='锁定到期时间') + last_attempt_time = models.DateTimeField(auto_now=True, verbose_name='最后尝试时间') + create_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间') + + class Meta: + db_table = 'login_attempt' + verbose_name = '登录尝试记录' + verbose_name_plural = verbose_name + unique_together = ['user', 'ip_address'] # 确保用户和IP组合唯一 + + def __str__(self): + return f"{self.user.username if self.user else 'Unknown'} - {self.ip_address}" \ No newline at end of file diff --git a/backend/apps/tests.py b/backend/apps/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/backend/apps/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/backend/apps/utils/auth.py b/backend/apps/utils/auth.py new file mode 100644 index 0000000..504120e --- /dev/null +++ b/backend/apps/utils/auth.py @@ -0,0 +1,56 @@ +import jwt +import logging +from functools import wraps +from django.http import JsonResponse +from django.conf import settings +from ..models import UserToken + +logger = logging.getLogger('apps') + +def jwt_auth_required(view_func): + @wraps(view_func) + def wrapper(request, *args, **kwargs): + token = request.headers.get('Authorization') + if not token: + logger.info('认证失败: 未提供Token') + return JsonResponse({ + 'code': 401, + 'message': '未提供Token' + }, status=401) + + try: + # 验证token是否存在于数据库 + user_token = UserToken.objects.filter(token=token).first() + if not user_token: + logger.info('认证失败: Token无效') + return JsonResponse({ + 'code': 401, + 'message': 'Token无效' + }, status=401) + + # 解析token + payload = jwt.decode(token, settings.SECRET_KEY, algorithms=['HS256']) + request.user_id = payload.get('user_id') + + return view_func(request, *args, **kwargs) + + except jwt.ExpiredSignatureError: + logger.info('认证失败: Token已过期') + return JsonResponse({ + 'code': 401, + 'message': 'Token已过期' + }, status=401) + except jwt.InvalidTokenError: + logger.info('认证失败: Token格式无效') + return JsonResponse({ + 'code': 401, + 'message': '无效的Token' + }, status=401) + except Exception as e: + logger.error('认证过程发生错误', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': '服务器错误' + }, status=500) + + return wrapper \ No newline at end of file diff --git a/backend/apps/utils/build_stages.py b/backend/apps/utils/build_stages.py new file mode 100644 index 0000000..dc9e1ac --- /dev/null +++ b/backend/apps/utils/build_stages.py @@ -0,0 +1,341 @@ +import os +import subprocess +import logging +import time +import tempfile +from typing import List, Dict, Any, Callable + +logger = logging.getLogger('apps') + +class BuildStageExecutor: + """构建阶段执行器""" + + def __init__(self, build_path: str, send_log: Callable, record_time: Callable): + """ + 初始化构建阶段执行器 + Args: + build_path: 构建目录路径 + send_log: 发送日志的回调函数 + record_time: 记录时间的回调函数 + """ + self.build_path = build_path + self.send_log = send_log + self.record_time = record_time + self.env = {} # 初始化为空字典，将由 Builder 设置 + + # 用于存储临时变量文件的路径 + self.vars_file = os.path.join(self.build_path, '.build_vars') + self._init_vars_file() + + def _init_vars_file(self): + """初始化变量文件""" + try: + with open(self.vars_file, 'w') as f: + f.write('#!/bin/bash\n# 构建变量\n') + # 设置执行权限 + os.chmod(self.vars_file, 0o755) + except Exception as e: + logger.error(f"初始化变量文件失败: {str(e)}", exc_info=True) + + def _save_variables_to_file(self, variables): + """ + 将变量保存到文件 + Args: + variables: 变量字典 + """ + try: + if not variables: + return + + # 追加变量到变量文件 + with open(self.vars_file, 'a') as f: + for name, value in variables.items(): + if name.startswith(('_', 'BASH_', 'SHELL', 'HOME', 'PATH', 'PWD', 'OLDPWD')): + continue + safe_value = self._escape_shell_value(str(value)) + f.write(f'export {name}={safe_value}\n') + except Exception as e: + logger.error(f"保存变量到文件失败: {str(e)}", exc_info=True) + + def _escape_shell_value(self, value): + """ + 转义shell变量值 + Args: + value: 要转义的值 + Returns: + str: 转义后的值 + """ + try: + # 如果值为空，返回空字符串 + if not value: + return '""' + + # 如果值只包含安全字符（字母、数字、下划线、点、斜杠、冒号），不需要引号 + import re + if re.match(r'^[a-zA-Z0-9_./:-]+$', value): + return value + + # 用单引号包围，并转义其中的单引号 + escaped_value = value.replace("'", "'\"'\"'") + return f"'{escaped_value}'" + except Exception as e: + logger.error(f"转义shell值失败: {str(e)}", exc_info=True) + return '""' + + def _is_variable_assignment(self, line): + """ + 检查是否是变量赋值语句 + Args: + line: 要检查的行 + Returns: + bool: 是否是变量赋值语句 + """ + try: + import re + # 去除前导空格 + line = line.strip() + + # 检查是否是 export VAR=value 格式 + if line.startswith('export '): + line = line[7:].strip() # 移除 'export ' 前缀 + + # 检查是否符合变量赋值格式：VAR=value + # 变量名必须以字母或下划线开头，后面可以跟字母、数字、下划线 + pattern = r'^[a-zA-Z_][a-zA-Z0-9_]*=' + return bool(re.match(pattern, line)) + except Exception as e: + logger.error(f"检查变量赋值语句失败: {str(e)}", exc_info=True) + return False + + def _create_temp_script_file(self, script_content: str, stage_name: str) -> str: + """ + 创建临时脚本文件，支持Jenkins风格的命令显示 + Args: + script_content: 内联脚本内容 + stage_name: 阶段名称 + Returns: + str: 临时脚本文件路径 + """ + try: + # 创建临时脚本文件 + with tempfile.NamedTemporaryFile( + mode='w', + suffix='.sh', + prefix=f'build_stage_{stage_name}_', + dir=self.build_path, + delete=False + ) as temp_file: + # 写入脚本头部 + temp_file.write('#!/bin/bash\n') + temp_file.write('set -e # 遇到错误立即退出\n') + temp_file.write('set -o pipefail # 管道命令中任何一个失败都视为失败\n') + temp_file.write('\n') + + # 加载变量文件 + temp_file.write(f'# 加载构建变量\n') + temp_file.write(f'source "{self.vars_file}" 2>/dev/null || true\n') + temp_file.write('\n') + + # 添加命令显示函数 + temp_file.write('''# 命令显示函数 +execute_with_display() { + echo "+ $*" + "$@" +} + +''') + + temp_file.write('# 用户脚本开始\n') + + # 逐行处理脚本内容 + for line in script_content.splitlines(): + line = line.strip() + if line and not line.startswith('#'): + # 检查是否是变量赋值语句（如：VAR=value 或 export VAR=value） + if self._is_variable_assignment(line): + # 变量赋值：直接显示并执行 + temp_file.write(f'echo "+ {line}"\n') + temp_file.write(f'{line}\n') + elif any(op in line for op in ['|', '>', '<', '&&', '||', ';']): + # 复杂命令：先显示，再在子shell中执行 + temp_file.write(f'echo "+ {line}"\n') + temp_file.write(f'bash -c {repr(line)}\n') + else: + # 简单命令：使用函数显示并执行 + temp_file.write(f'execute_with_display {line}\n') + elif line.startswith('#'): + # 保留注释 + temp_file.write(f'{line}\n') + + temp_file.write('\n') + + script_path = temp_file.name + + # 设置执行权限 + os.chmod(script_path, 0o755) + return script_path + + except Exception as e: + self.send_log(f"创建临时脚本文件失败: {str(e)}", stage_name) + return None + + def _execute_script_unified(self, script_path: str, stage_name: str, check_termination: Callable = None) -> bool: + try: + # 检查是否应该终止 + if check_termination and check_termination(): + self.send_log("构建已被终止，跳过脚本执行", stage_name) + return False + + # 执行脚本，合并stdout和stderr到同一个流，保持输出顺序 + process = subprocess.Popen( + ['/bin/bash', script_path], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # 将stderr重定向到stdout，保持输出顺序 + cwd=self.build_path, + env=self.env, + universal_newlines=True, + bufsize=1 # 行缓冲，确保输出能够实时获取 + ) + + # 实时读取并发送输出 + import fcntl + import os + + # 设置非阻塞模式 + fd = process.stdout.fileno() + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) + + # 持续读取直到进程结束 + while process.poll() is None: + # 检查是否终止 + if check_termination and check_termination(): + process.terminate() + self.send_log("构建已被终止，停止当前脚本", stage_name) + return False + + try: + line = process.stdout.readline() + if line: + line = line.rstrip() + self.send_log(line, stage_name, raw_output=True) + except BlockingIOError: + time.sleep(0.01) + continue + + remaining_output = process.stdout.read() + if remaining_output: + for line in remaining_output.splitlines(): + if line.strip(): + self.send_log(line.rstrip(), stage_name, raw_output=True) + + # 检查执行结果 + success = process.returncode == 0 + + if not success: + self.send_log(f"脚本执行失败，返回码: {process.returncode}", stage_name) + + return success + + except Exception as e: + self.send_log(f"执行脚本时发生错误: {str(e)}", stage_name) + return False + finally: + if script_path.startswith(tempfile.gettempdir()) or '/build_stage_' in script_path: + try: + os.unlink(script_path) + except Exception as e: + logger.debug(f"清理临时脚本文件失败: {str(e)}") + + def execute_stage(self, stage: Dict[str, Any], check_termination: Callable = None) -> bool: + """ + 执行单个构建阶段 + Args: + stage: 阶段配置信息 + check_termination: 检查是否终止的回调函数 + Returns: + bool: 执行是否成功 + """ + try: + stage_name = stage.get('name', '未命名阶段') + + if check_termination and check_termination(): + self.send_log("构建已被终止，跳过此阶段", stage_name) + return False + + # 记录阶段开始时间 + stage_start_time = time.time() + + # 执行脚本 + success = self._execute_inline_script(stage, check_termination) + + # 记录阶段耗时 + stage_duration = time.time() - stage_start_time + self.record_time(stage_name, stage_start_time, stage_duration) + + return success + + except Exception as e: + self.send_log(f"执行阶段时发生错误: {str(e)}", stage_name) + return False + + def _execute_inline_script(self, stage: Dict[str, Any], check_termination: Callable = None) -> bool: + """ + 执行内联脚本 + Args: + stage: 阶段配置信息 + check_termination: 检查是否终止的回调函数 + Returns: + bool: 执行是否成功 + """ + stage_name = stage.get('name', '未命名阶段') + try: + script_content = stage.get('script', '').strip() + if not script_content: + self.send_log("脚本内容为空", stage_name) + return False + + # 创建临时脚本文件 + script_path = self._create_temp_script_file(script_content, stage_name) + if not script_path: + return False + + # 脚本执行方法 + success = self._execute_script_unified(script_path, stage_name, check_termination) + + return success + + except Exception as e: + self.send_log(f"执行内联脚本时发生错误: {str(e)}", stage_name) + return False + + def execute_stages(self, stages: List[Dict[str, Any]], check_termination: Callable = None) -> bool: + """ + 执行所有构建阶段 + Args: + stages: 阶段配置列表 + check_termination: 检查是否终止的回调函数 + Returns: + bool: 所有阶段是否都执行成功 + """ + if not stages: + self.send_log("没有配置构建阶段") + return False + + for stage in stages: + if check_termination and check_termination(): + self.send_log("构建已被终止，跳过后续阶段", "Build Stages") + return False + + stage_name = stage.get('name', '未命名阶段') + self.send_log(f"开始执行阶段: {stage_name}", "Build Stages") + + # 执行当前阶段 + if not self.execute_stage(stage, check_termination): + self.send_log(f"阶段 {stage_name} 执行失败", "Build Stages") + return False + + self.send_log(f"阶段 {stage_name} 执行完成", "Build Stages") + + self.send_log("所有阶段执行完成", "Build Stages") + return True \ No newline at end of file diff --git a/backend/apps/utils/builder.py b/backend/apps/utils/builder.py new file mode 100644 index 0000000..a119db9 --- /dev/null +++ b/backend/apps/utils/builder.py @@ -0,0 +1,578 @@ +import os +import logging +import time +import subprocess +import tempfile +import re +import shutil +from datetime import datetime +from pathlib import Path +from django.conf import settings +from git import Repo +from git.exc import GitCommandError +from .build_stages import BuildStageExecutor +from .notifier import BuildNotifier +from .log_stream import log_stream_manager +from django.db.models import F +from ..models import BuildTask, BuildHistory +# from ..utils.builder import Builder +# from ..utils.crypto import decrypt_sensitive_data + +logger = logging.getLogger('apps') + +class Builder: + def __init__(self, task, build_number, commit_id, history): + self.task = task + self.build_number = build_number + self.commit_id = commit_id + self.history = history # 构建历史记录 + self.log_buffer = [] # 缓存日志 + + # 检查是否已有指定的版本号 + if self.history.version: + self.version = self.history.version + self.send_log(f"使用指定版本: {self.version}", "Version") + else: + # 为开发和测试环境生成新的版本号 + self.version = f"{datetime.now().strftime('%Y%m%d%H%M%S')}_{commit_id[:8]}" + + # 初始化构建时间信息 + self.build_time = { + 'start_time': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + 'stages_time': [] + } + + # 更新构建历史记录的状态和版本 + self.history.status = 'running' + if not self.history.version: # 只有当没有版本号时才更新 + self.history.version = self.version + self.history.build_time = self.build_time + self.history.save(update_fields=['status', 'version', 'build_time']) + + # 设置构建目录 + self.build_path = Path(settings.BUILD_ROOT) / task.name / self.version / task.project.name + + # 创建实时日志流 + log_stream_manager.create_build_stream(self.task.task_id, self.build_number) + + def check_if_terminated(self): + """检查构建是否已被终止""" + # 从数据库重新加载构建历史记录，以获取最新状态 + try: + history_record = BuildHistory.objects.get(history_id=self.history.history_id) + if history_record.status == 'terminated': + # 如果状态为terminated，构建已被手动终止 + self.send_log("检测到构建已被手动终止，停止后续步骤", "System") + return True + return False + except Exception as e: + logger.error(f"检查构建状态时出错: {str(e)}", exc_info=True) + return False + + def _filter_maven_progress(self, message): + # 只过滤Maven下载进度信息中的Progress()部分 + if 'Progress (' in message and ('KB' in message or 'MB' in message or 'B/s' in message): + return None + + # 过滤Maven下载进度条 + if re.match(r'^Progress \(\d+\): .+', message.strip()): + return None + + # 过滤空的进度行 + if re.match(r'^\s*Progress\s*$', message.strip()): + return None + + # 过滤下载进度百分比 + if re.match(r'^\s*\d+%\s*$', message.strip()): + return None + + return message + + def send_log(self, message, stage=None, console_only=False, raw_output=False): + """发送日志到缓存、实时流和控制台 + Args: + message: 日志消息 + stage: 阶段名称 + console_only: 是否只输出到控制台（保留参数兼容性） + raw_output: 是否为原始输出（不添加阶段标记） + """ + # 过滤Maven Progress信息 + filtered_message = self._filter_maven_progress(message) + if filtered_message is None: + return # 跳过被过滤的消息 + + # 格式化消息 + if raw_output: + formatted_message = filtered_message + else: + formatted_message = filtered_message + + # 如果有阶段名称，添加阶段标记 + if stage: + formatted_message = f"[{stage}] {filtered_message}" + + # 缓存日志 + self.log_buffer.append(formatted_message) + + # 推送到实时日志流 + try: + log_stream_manager.push_log( + task_id=self.task.task_id, + build_number=self.build_number, + message=formatted_message + '\n', + stage=stage + ) + except Exception as e: + # 降低日志级别，避免在清理阶段产生过多错误日志 + if "日志队列不存在" in str(e): + logger.debug(f"日志队列已清理，跳过推送: {str(e)}") + else: + logger.error(f"推送实时日志失败: {str(e)}", exc_info=True) + + # 批量更新数据库中的构建日志 + try: + should_update_db = ( + len(self.log_buffer) % 10 == 0 or # 每10条日志更新一次 + not hasattr(self, '_last_db_update') or + time.time() - getattr(self, '_last_db_update', 0) >= 5 # 每5秒更新一次 + ) + + if should_update_db: + current_log = '\n'.join(self.log_buffer) + self.history.build_log = current_log + self.history.save(update_fields=['build_log']) + self._last_db_update = time.time() + except Exception as e: + logger.error(f"批量更新构建日志失败: {str(e)}", exc_info=True) + + # 输出到控制台 - 确保构建日志在控制台显示 + logger.info(formatted_message, extra={ + 'from_builder': True, # 添加标记以区分构建日志 + 'task_id': self.task.task_id, + 'build_number': self.build_number + }) + + def _save_build_log(self): + """保存构建日志到历史记录""" + try: + self.history.build_log = '\n'.join(self.log_buffer) + self.history.save(update_fields=['build_log']) + except Exception as e: + logger.error(f"保存构建日志失败: {str(e)}", exc_info=True) + + def clone_repository(self): + """克隆Git仓库""" + try: + # 检查构建是否已被终止 + if self.check_if_terminated(): + return False + + self.send_log("开始克隆代码...", "Git Clone") + self.send_log(f"构建目录: {self.build_path}", "Git Clone") + + # 确保目录存在 + self.build_path.parent.mkdir(parents=True, exist_ok=True) + + # 获取Git凭证 + repository = self.task.project.repository + self.send_log(f"仓库地址: {repository}", "Git Clone") + git_token = self.task.git_token.token if self.task.git_token else None + + # 处理带有token的仓库URL + if git_token and repository.startswith('http'): + if '@' in repository: + repository = repository.split('@')[1] + repository = f'https://oauth2:{git_token}@{repository}' + else: + repository = repository.replace('://', f'://oauth2:{git_token}@') + + # 使用构建历史记录中的分支 + branch = self.history.branch + self.send_log(f"克隆分支: {branch}", "Git Clone") + self.send_log("正在克隆代码，请稍候...", "Git Clone") + + # 克隆指定分支的代码 + Repo.clone_from( + repository, + str(self.build_path), + branch=branch, + progress=self.git_progress + ) + + # 检查构建是否已被终止 + if self.check_if_terminated(): + return False + + # 验证克隆是否成功 + if not os.path.exists(self.build_path) or not os.listdir(self.build_path): + self.send_log("代码克隆失败：目录为空", "Git Clone") + return False + + self.send_log("代码克隆完成", "Git Clone") + self.send_log(f"克隆目录验证成功: {self.build_path}", "Git Clone") + return True + + except GitCommandError as e: + self.send_log(f"克隆代码失败: {str(e)}", "Git Clone") + return False + except Exception as e: + self.send_log(f"发生错误: {str(e)}", "Git Clone") + return False + + def git_progress(self, op_code, cur_count, max_count=None, message=''): + """Git进度回调""" + # 每秒检查一次构建是否已被终止 + if int(time.time()) % 5 == 0: # 每5秒检查一次 + if self.check_if_terminated(): + # 如果构建已被终止，尝试引发异常停止Git克隆 + raise Exception("Build terminated") + pass + + def clone_external_scripts(self): + """克隆外部脚本库""" + try: + if not self.task.use_external_script: + return True + + # 检查外部脚本库配置 + config = self.task.external_script_config + if not config or not config.get('repo_url') or not config.get('directory') or not config.get('branch'): + self.send_log("外部脚本库配置不完整，跳过克隆", "External Scripts") + return True + + # 检查构建是否已被终止 + if self.check_if_terminated(): + return False + + repo_url = config.get('repo_url') + base_directory = config.get('directory') + branch = config.get('branch') # 分支为必填项 + token_id = config.get('token_id') + + # 从仓库URL中提取项目名称 + import re + repo_name_match = re.search(r'/([^/]+?)(?:\.git)?/?$', repo_url) + if repo_name_match: + repo_name = repo_name_match.group(1) + if repo_name.endswith('.git'): + repo_name = repo_name[:-4] + else: + repo_name = 'external-scripts' + + # 完整的克隆目录路径 + directory = os.path.join(base_directory, repo_name) + + self.send_log("开始克隆外部脚本库...", "External Scripts") + self.send_log(f"仓库地址: {repo_url}", "External Scripts") + self.send_log(f"基础目录: {base_directory}", "External Scripts") + self.send_log(f"项目名称: {repo_name}", "External Scripts") + self.send_log(f"完整目录: {directory}", "External Scripts") + self.send_log(f"分支: {branch}", "External Scripts") + + # 获取Git Token（如果配置了） + git_token = None + if token_id: + try: + from ..models import GitlabTokenCredential + credential = GitlabTokenCredential.objects.get(credential_id=token_id) + git_token = credential.token + except: + self.send_log("获取Git Token失败，尝试使用公开仓库方式克隆", "External Scripts") + + # 处理带有token的仓库URL + if git_token and repo_url.startswith('http'): + if '@' in repo_url: + repo_url = repo_url.split('@')[1] + repo_url = f'https://oauth2:{git_token}@{repo_url}' + else: + repo_url = repo_url.replace('://', f'://oauth2:{git_token}@') + + # 确保基础目录存在 + os.makedirs(base_directory, exist_ok=True) + + # 如果目标目录已存在且不为空，先清空 + if os.path.exists(directory) and os.listdir(directory): + self.send_log(f"清空现有目录: {directory}", "External Scripts") + shutil.rmtree(directory) + + # 克隆外部脚本库 + self.send_log("正在克隆外部脚本库，请稍候...", "External Scripts") + + from git import Repo + # 使用指定分支克隆 + Repo.clone_from( + repo_url, + directory, + branch=branch + ) + + # 再次检查构建是否已被终止 + if self.check_if_terminated(): + return False + + # 验证克隆是否成功 + if not os.path.exists(directory) or not os.listdir(directory): + self.send_log("外部脚本库克隆失败：目录为空", "External Scripts") + return False + + self.send_log("外部脚本库克隆完成", "External Scripts") + self.send_log(f"克隆目录验证成功: {directory}", "External Scripts") + return True + + except Exception as e: + self.send_log(f"克隆外部脚本库失败: {str(e)}", "External Scripts") + # 如果用户配置了外部脚本库，克隆失败应该终止构建 + self.send_log("外部脚本库克隆失败，终止构建", "External Scripts") + return False + + def execute_stages(self, stage_executor): + """执行构建阶段""" + try: + if not self.task.stages: + self.send_log("没有配置构建阶段", "Build Stages") + return False + + # 检查构建是否已被终止 + if self.check_if_terminated(): + return False + + # 执行所有阶段 + success = stage_executor.execute_stages(self.task.stages, check_termination=self.check_if_terminated) + return success + + except Exception as e: + self.send_log(f"执行构建阶段时发生错误: {str(e)}", "Build Stages") + return False + + def execute(self): + """执行构建""" + build_start_time = time.time() + success = False # 初始化成功状态 + try: + # 在开始构建前检查构建是否已被终止 + if self.check_if_terminated(): + self._update_build_stats(False) + self._save_build_log() + return False + + # 获取环境类型 + environment_type = self.task.environment.type if self.task.environment else None + + # 根据环境类型决定是否需要克隆代码 + if environment_type in ['development', 'testing']: + # 只在开发和测试环境克隆代码 + clone_start_time = time.time() + if not self.clone_repository(): + self._update_build_stats(False) # 更新失败统计 + self._update_build_time(build_start_time, False) + # 发送构建失败通知 + notifier = BuildNotifier(self.history) + notifier.send_notifications() + return False + + # 记录代码克隆阶段的时间 + self.build_time['stages_time'].append({ + 'name': 'Git Clone', + 'start_time': datetime.fromtimestamp(clone_start_time).strftime('%Y-%m-%d %H:%M:%S'), + 'duration': str(int(time.time() - clone_start_time)) + }) + else: + self.send_log(f"预发布/生产环境构建，跳过代码克隆，直接使用版本: {self.version}", "Environment") + + # 创建构建目录 + os.makedirs(self.build_path, exist_ok=True) + + # 再次检查构建是否已被终止 + if self.check_if_terminated(): + self._update_build_stats(False) + self._update_build_time(build_start_time, False) + return False + + # 克隆外部脚本库（如果配置了） + external_script_start_time = time.time() + if not self.clone_external_scripts(): + self._update_build_stats(False) + self._update_build_time(build_start_time, False) + # 发送构建失败通知 + notifier = BuildNotifier(self.history) + notifier.send_notifications() + return False + + # 记录外部脚本库克隆阶段的时间（如果启用了外部脚本库） + if self.task.use_external_script: + self.build_time['stages_time'].append({ + 'name': 'External Scripts Clone', + 'start_time': datetime.fromtimestamp(external_script_start_time).strftime('%Y-%m-%d %H:%M:%S'), + 'duration': str(int(time.time() - external_script_start_time)) + }) + + # 检查构建是否已被终止 + if self.check_if_terminated(): + self._update_build_stats(False) + self._update_build_time(build_start_time, False) + return False + + # 创建阶段执行器，传递send_log方法和构建时间记录回调 + stage_executor = BuildStageExecutor( + str(self.build_path), + lambda msg, stage=None, raw_output=False: self.send_log(msg, stage, raw_output=raw_output), + self._record_stage_time + ) + + # 设置系统内置环境变量 + system_variables = { + # 编号相关变量 + 'BUILD_NUMBER': str(self.build_number), + 'VERSION': self.version, + + # Git相关变量 + 'COMMIT_ID': self.commit_id, + 'BRANCH': self.history.branch, + + # 项目相关变量 + 'PROJECT_NAME': self.task.project.name, + 'PROJECT_ID': self.task.project.project_id, + 'PROJECT_REPO': self.task.project.repository, + + # 任务相关变量 + 'TASK_NAME': self.task.name, + 'TASK_ID': self.task.task_id, + + # 环境相关变量 + 'ENVIRONMENT': self.task.environment.name, + 'ENVIRONMENT_TYPE': self.task.environment.type, + 'ENVIRONMENT_ID': self.task.environment.environment_id, + + # 别名(便于使用) + 'service_name': self.task.name, + 'build_env': self.task.environment.name, + 'branch': self.history.branch, + 'version': self.version, + + # 构建路径 + 'BUILD_PATH': str(self.build_path), + 'BUILD_WORKSPACE': str(self.build_path), + + # Docker配置 + 'DOCKER_BUILDKIT': '0', + 'BUILDKIT_PROGRESS': 'plain', + + # Locale配置 - 使用稳定的POSIX locale避免SSH连接时的警告 + 'LC_ALL': 'POSIX', + 'LANG': 'POSIX', + } + + combined_env = {**os.environ, **system_variables} + stage_executor.env = combined_env + + # 保存系统变量到文件 + stage_executor._save_variables_to_file(system_variables) + + # 执行构建阶段 + success = self.execute_stages(stage_executor) + return success + + except Exception as e: + self.send_log(f"构建过程中发生未捕获的异常: {str(e)}", "Error") + success = False + return False + finally: + # 更新构建统计和时间信息 + self._update_build_stats(success) + self._update_build_time(build_start_time, success) + + # 确保最终日志保存到数据库 + self._save_build_log() + + # 输出构建完成状态日志 + self.history.refresh_from_db() + final_status = self.history.status + self.send_log(f"构建完成，状态: {final_status}", "Build") + + # 确保构建完成状态日志也保存到数据库 + self._save_build_log() + + # 通知日志流管理器构建完成 + try: + log_stream_manager.complete_build( + task_id=self.task.task_id, + build_number=self.build_number, + status=final_status + ) + except Exception as e: + logger.error(f"通知日志流管理器构建完成失败: {str(e)}", exc_info=True) + + # 发送构建通知 + notifier = BuildNotifier(self.history) + notifier.send_notifications() + + def _record_stage_time(self, stage_name: str, start_time: float, duration: float): + """记录阶段执行时间 + Args: + stage_name: 阶段名称 + start_time: 开始时间戳 + duration: 耗时（秒） + """ + stage_time = { + 'name': stage_name, + 'start_time': datetime.fromtimestamp(start_time).strftime('%Y-%m-%d %H:%M:%S'), + 'duration': str(int(duration)) + } + self.build_time['stages_time'].append(stage_time) + + # 更新构建历史记录的阶段信息 + self.history.stages = self.task.stages + self.history.save(update_fields=['stages']) + + def _update_build_time(self, build_start_time: float, success: bool): + """更新构建时间信息 + Args: + build_start_time: 构建开始时间戳 + success: 构建是否成功 + """ + try: + # 计算总耗时 + total_duration = int(time.time() - build_start_time) + + # 更新构建时间信息 + self.build_time['total_duration'] = str(total_duration) + self.build_time['end_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + # 检查当前构建状态，如果已经是terminated则不覆盖状态 + self.history.refresh_from_db() + if self.history.status != 'terminated': + # 只有在状态不是terminated时才更新状态 + self.history.status = 'success' if success else 'failed' + + self.history.build_time = self.build_time + self.history.save(update_fields=['status', 'build_time']) + except Exception as e: + logger.error(f"更新构建时间信息失败: {str(e)}", exc_info=True) + + def _update_build_stats(self, success: bool): + """更新构建统计信息 + Args: + success: 构建是否成功 + """ + try: + # 检查当前构建状态，如果是terminated则不更新统计 + self.history.refresh_from_db() + if self.history.status == 'terminated': + return + + # 更新任务的构建统计信息 + if success: + BuildTask.objects.filter(task_id=self.task.task_id).update( + success_builds=F('success_builds') + 1 + ) + # 只有成功的构建才更新版本号 + BuildTask.objects.filter(task_id=self.task.task_id).update( + version=self.version + ) + else: + BuildTask.objects.filter(task_id=self.task.task_id).update( + failure_builds=F('failure_builds') + 1 + ) + except Exception as e: + logger.error(f"更新构建统计信息失败: {str(e)}", exc_info=True) + diff --git a/backend/apps/utils/log_stream.py b/backend/apps/utils/log_stream.py new file mode 100644 index 0000000..599a2c6 --- /dev/null +++ b/backend/apps/utils/log_stream.py @@ -0,0 +1,194 @@ +import threading +import queue +import time +import logging +from typing import Dict, Set, Optional +from dataclasses import dataclass + +logger = logging.getLogger('apps') + +@dataclass +class LogMessage: + """日志消息数据类""" + task_id: str + build_number: int + message: str + stage: Optional[str] = None + timestamp: float = None + + def __post_init__(self): + if self.timestamp is None: + self.timestamp = time.time() + +class LogStreamManager: + + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if hasattr(self, '_initialized'): + return + + self._initialized = True + # 存储每个构建的日志队列 {(task_id, build_number): queue.Queue} + self._log_queues: Dict[tuple, queue.Queue] = {} + # 存储每个构建的SSE客户端集合 {(task_id, build_number): set} + self._sse_clients: Dict[tuple, Set] = {} + # 线程锁 + self._queues_lock = threading.Lock() + self._clients_lock = threading.Lock() + + logger.info("LogStreamManager initialized") + + def get_build_key(self, task_id: str, build_number: int) -> tuple: + """获取构建的唯一键""" + return (task_id, build_number) + + def create_build_stream(self, task_id: str, build_number: int): + """为新构建创建日志流""" + build_key = self.get_build_key(task_id, build_number) + + with self._queues_lock: + if build_key not in self._log_queues: + self._log_queues[build_key] = queue.Queue(maxsize=20000) # 最大缓存20000条日志 + logger.info(f"Created log stream for build {task_id}#{build_number}") + + with self._clients_lock: + if build_key not in self._sse_clients: + self._sse_clients[build_key] = set() + + def add_sse_client(self, task_id: str, build_number: int, client_id: str): + """添加SSE客户端""" + build_key = self.get_build_key(task_id, build_number) + + with self._clients_lock: + if build_key not in self._sse_clients: + self._sse_clients[build_key] = set() + self._sse_clients[build_key].add(client_id) + logger.info(f"Added SSE client {client_id} for build {task_id}#{build_number}") + + def remove_sse_client(self, task_id: str, build_number: int, client_id: str): + """移除SSE客户端""" + build_key = self.get_build_key(task_id, build_number) + + with self._clients_lock: + if build_key in self._sse_clients: + self._sse_clients[build_key].discard(client_id) + logger.info(f"Removed SSE client {client_id} for build {task_id}#{build_number}") + + if not self._sse_clients[build_key]: + del self._sse_clients[build_key] + self._cleanup_build_stream(build_key) + + def _cleanup_build_stream(self, build_key: tuple): + """清理构建流资源""" + with self._queues_lock: + if build_key in self._log_queues: + del self._log_queues[build_key] + logger.info(f"Cleaned up log stream for build {build_key[0]}#{build_key[1]}") + + def push_log(self, task_id: str, build_number: int, message: str, stage: Optional[str] = None): + """推送日志消息到流""" + build_key = self.get_build_key(task_id, build_number) + + # 创建日志消息 + log_msg = LogMessage( + task_id=task_id, + build_number=build_number, + message=message, + stage=stage + ) + + # 推送到队列 + with self._queues_lock: + if build_key not in self._log_queues: + # 如果队列不存在，先创建 + self._log_queues[build_key] = queue.Queue(maxsize=20000) + logger.debug(f"Created log queue for build {task_id}#{build_number} during push") + + try: + # 非阻塞推送，如果队列满了就丢弃最老的消息 + if self._log_queues[build_key].full(): + try: + self._log_queues[build_key].get_nowait() # 移除最老的消息 + except queue.Empty: + pass + + self._log_queues[build_key].put_nowait(log_msg) + except queue.Full: + logger.warning(f"Log queue full for build {task_id}#{build_number}, dropping message") + + def get_log_stream(self, task_id: str, build_number: int, client_id: str): + """获取日志流生成器""" + build_key = self.get_build_key(task_id, build_number) + + # 确保流存在 + self.create_build_stream(task_id, build_number) + self.add_sse_client(task_id, build_number, client_id) + + try: + while True: + try: + # 检查客户端是否还在连接 + with self._clients_lock: + if build_key not in self._sse_clients or client_id not in self._sse_clients[build_key]: + logger.info(f"Client {client_id} disconnected from build {task_id}#{build_number}") + break + + # 获取日志队列 + with self._queues_lock: + log_queue = self._log_queues.get(build_key) + + if log_queue is None: + break + + try: + # 等待新日志消息，超时时间为1秒 + log_msg = log_queue.get(timeout=1.0) + yield log_msg + except queue.Empty: + # 超时，发送心跳 + yield None # None表示心跳 + + except Exception as e: + logger.error(f"Error in log stream for {task_id}#{build_number}: {str(e)}") + break + finally: + # 清理客户端 + self.remove_sse_client(task_id, build_number, client_id) + + def complete_build(self, task_id: str, build_number: int, status: str): + """标记构建完成""" + build_key = self.get_build_key(task_id, build_number) + + # 推送完成消息 + completion_msg = LogMessage( + task_id=task_id, + build_number=build_number, + message=f"BUILD_COMPLETE:{status}", + stage="SYSTEM" + ) + + with self._queues_lock: + if build_key in self._log_queues: + try: + self._log_queues[build_key].put_nowait(completion_msg) + except queue.Full: + pass + + def has_active_clients(self, task_id: str, build_number: int) -> bool: + """检查是否有活跃的SSE客户端""" + build_key = self.get_build_key(task_id, build_number) + + with self._clients_lock: + return build_key in self._sse_clients and len(self._sse_clients[build_key]) > 0 + +# 全局单例实例 +log_stream_manager = LogStreamManager() \ No newline at end of file diff --git a/backend/apps/utils/notifier.py b/backend/apps/utils/notifier.py new file mode 100644 index 0000000..e00fe8b --- /dev/null +++ b/backend/apps/utils/notifier.py @@ -0,0 +1,252 @@ +import json +import time +import hmac +import base64 +import hashlib +import logging +import requests +from urllib.parse import quote_plus +from django.conf import settings +from ..models import NotificationRobot, BuildHistory + +logger = logging.getLogger('apps') + +class BuildNotifier: + """构建通知工具类""" + + def __init__(self, history: BuildHistory): + self.history = history + self.task = history.task + self.project = history.task.project + self.environment = history.task.environment + + def _sign_dingtalk(self, secret: str, timestamp: str) -> str: + """钉钉机器人签名""" + string_to_sign = f'{timestamp}\n{secret}' + hmac_code = hmac.new( + secret.encode('utf-8'), + string_to_sign.encode('utf-8'), + digestmod=hashlib.sha256 + ).digest() + return base64.b64encode(hmac_code).decode('utf-8') + + def _sign_feishu(self, secret: str, timestamp: str) -> str: + """飞书机器人签名""" + string_to_sign = f'{timestamp}\n{secret}' + hmac_code = hmac.new( + secret.encode('utf-8'), + string_to_sign.encode('utf-8'), + digestmod=hashlib.sha256 + ).digest() + return base64.b64encode(hmac_code).decode('utf-8') + + def _get_build_status_emoji(self) -> str: + """获取构建状态对应的emoji""" + status_emoji = { + 'success': '✅', + 'failed': '❌', + 'running': '🔄', + 'pending': '⏳', + 'terminated': '🛑' + } + return status_emoji.get(self.history.status, '❓') + + def _get_duration_text(self) -> str: + """获取构建耗时文本""" + if not self.history.build_time or 'total_duration' not in self.history.build_time: + return '未完成' + + duration = int(self.history.build_time['total_duration']) + if duration < 60: + return f'{duration}秒' + minutes = duration // 60 + seconds = duration % 60 + return f'{minutes}分{seconds}秒' + + def _get_status_text(self) -> str: + """获取状态文本""" + status_texts = { + 'success': '构建成功', + 'failed': '构建失败', + 'running': '构建中', + 'pending': '等待中', + 'terminated': '构建已终止' + } + return status_texts.get(self.history.status, self.history.status) + + def _get_build_url(self) -> str: + """获取构建历史页面URL""" + base_url = getattr(settings, 'WEB_URL', 'http://localhost:5173') # 如果未配置，使用默认值 + return f"{base_url}/build/history?history_id={self.history.history_id}" + + def _format_dingtalk_message(self) -> dict: + """格式化钉钉通知消息""" + status_text = self._get_status_text() + build_url = self._get_build_url() + + content = [ + f"## 🔔 构建通知：{status_text}", + "---", + "**构建详情：**", + f"- **任务名称**：{self.task.name}", + f"- **构建编号**：#{self.history.build_number}", + f"- **构建版本**：{self.history.version}", + f"- **构建分支**：{self.history.branch}", + f"- **提交ID**：{self.history.commit_id[:8] if self.history.commit_id else '无'}", + f"- **构建环境**：{self.environment.name}", + f"- **构建人员**：{self.history.operator.name if self.history.operator else '系统'}", + f"- **构建耗时**：{self._get_duration_text()}", + "", + "**构建需求：**", + f"> {self.history.requirement or '无'}", + "", + f"**查看详情：**[点击查看构建日志]({build_url})", + "", + "---", + "**注意事项：**", + "1. 此为自动通知，请勿回复", + "2. 如遇构建失败，请先查看构建日志进行排查", + "3. 如需帮助，请联系运维同学" + ] + + return { + "msgtype": "markdown", + "markdown": { + "title": f"{status_text}: {self.task.name} #{self.history.build_number}", + "text": "\n".join(content) + }, + "at": { + "isAtAll": True + } + } + + def _format_wecom_message(self) -> dict: + """格式化企业微信通知消息""" + status_text = self._get_status_text() + build_url = self._get_build_url() + + content = [ + f"## 🔔 构建通知：{status_text}", + "---", + "@all", # 企业微信使用 @all 来@所有人 + "", + "**构建详情：**", + f"- **任务名称**：{self.task.name}", + f"- **构建编号**：#{self.history.build_number}", + f"- **构建版本**：{self.history.version}", + f"- **构建分支**：{self.history.branch}", + f"- **提交ID**：{self.history.commit_id[:8] if self.history.commit_id else '无'}", + f"- **构建环境**：{self.environment.name}", + f"- **构建人员**：{self.history.operator.name if self.history.operator else '系统'}", + f"- **构建耗时**：{self._get_duration_text()}", + "", + "**构建需求：**", + f"> {self.history.requirement or '无'}", + "", + f"**查看详情：**[点击查看构建日志]({build_url})", + "", + "---", + "**注意事项：**", + "1. 此为自动通知，请勿回复", + "2. 如遇构建失败，请先查看构建日志进行排查", + "3. 如需帮助，请联系运维同学" + ] + + return { + "msgtype": "markdown", + "markdown": { + "content": "\n".join(content) + } + } + + def _format_feishu_message(self) -> dict: + """格式化飞书通知消息""" + status_text = self._get_status_text() + build_url = self._get_build_url() + + content = [ + f"🔔 构建通知：{status_text}", + "---", + "所有人", # 飞书使用这种格式@所有人 + "", + "**构建详情：**", + f"- **任务名称**：{self.task.name}", + f"- **构建编号**：#{self.history.build_number}", + f"- **构建版本**：{self.history.version}", + f"- **构建分支**：{self.history.branch}", + f"- **提交ID**：{self.history.commit_id[:8] if self.history.commit_id else '无'}", + f"- **构建环境**：{self.environment.name}", + f"- **构建人员**：{self.history.operator.name if self.history.operator else '系统'}", + f"- **构建耗时**：{self._get_duration_text()}", + "", + "**构建需求：**", + f"> {self.history.requirement or '无'}", + "", + f"**查看详情：**[点击查看构建日志]({build_url})", + "", + "---", + "**注意事项：**", + "1. 此为自动通知，请勿回复", + "2. 如遇构建失败，请先查看构建日志进行排查", + "3. 如需帮助，请联系运维同学" + ] + + return { + "msg_type": "text", + "content": { + "text": "\n".join(content) + } + } + + def send_notifications(self): + """发送构建通知""" + if not self.task.notification_channels: + logger.info(f"任务 {self.task.name} 未配置通知方式") + return + + # 获取需要通知的机器人 + robots = NotificationRobot.objects.filter(robot_id__in=self.task.notification_channels) + + for robot in robots: + try: + webhook = robot.webhook + timestamp = str(int(time.time() * 1000)) + headers = {} + + # 根据机器人类型处理安全设置 + if robot.security_type == 'secret' and robot.secret: + if robot.type == 'dingtalk': + sign = self._sign_dingtalk(robot.secret, timestamp) + webhook = f"{webhook}×tamp={timestamp}&sign={quote_plus(sign)}" + elif robot.type == 'feishu': + sign = self._sign_feishu(robot.secret, timestamp) + headers.update({ + "X-Timestamp": timestamp, + "X-Sign": sign + }) + + # 根据机器人类型获取消息内容 + if robot.type == 'dingtalk': + message = self._format_dingtalk_message() + elif robot.type == 'wecom': + message = self._format_wecom_message() + elif robot.type == 'feishu': + message = self._format_feishu_message() + else: + logger.error(f"不支持的机器人类型: {robot.type}") + continue + + # 发送通知 + response = requests.post(webhook, json=message, headers=headers) + + if response.status_code == 200: + resp_json = response.json() + if resp_json.get('errcode') == 0 or resp_json.get('StatusCode') == 0 or resp_json.get('code') == 0: + logger.info(f"发送 {robot.type} 通知成功: {robot.name}") + else: + logger.error(f"发送 {robot.type} 通知失败: {response.text}") + else: + logger.error(f"发送 {robot.type} 通知失败: {response.text}") + + except Exception as e: + logger.error(f"发送 {robot.type} 通知出错: {str(e)}", exc_info=True) \ No newline at end of file diff --git a/backend/apps/utils/permissions.py b/backend/apps/utils/permissions.py new file mode 100644 index 0000000..871d91f --- /dev/null +++ b/backend/apps/utils/permissions.py @@ -0,0 +1,128 @@ +import json +import logging +from ..models import User, UserRole + +logger = logging.getLogger('apps') + +def get_user_permissions(user_id): + """ + 获取用户的权限信息 + + Args: + user_id: 用户ID + + Returns: + dict: 用户权限信息，包含菜单权限、功能权限和数据权限 + """ + try: + # 获取用户信息 + try: + user = User.objects.get(user_id=user_id) + except User.DoesNotExist: + logger.error(f'获取用户权限时用户不存在: {user_id}') + return { + 'menu': [], + 'function': {}, + 'data': { + 'project_scope': 'all', + 'project_ids': [], + 'environment_scope': 'all', + 'environment_types': [] + } + } + + # 获取用户的所有角色 + user_roles = UserRole.objects.filter(user=user).select_related('role') + + # 合并所有角色的权限 + menu_permissions = set() + function_permissions = {} + data_permissions = { + 'project_scope': 'all', + 'project_ids': [], + 'environment_scope': 'all', + 'environment_types': [] + } + + has_custom_project_scope = False + has_custom_environment_scope = False + + for user_role in user_roles: + role = user_role.role + permissions = role.permissions + + # 如果permissions是字符串，解析为JSON + if isinstance(permissions, str): + try: + permissions = json.loads(permissions) + except: + permissions = {} + + # 合并菜单权限 + if permissions.get('menu'): + menu_permissions.update(permissions['menu']) + + # 合并功能权限 + if permissions.get('function'): + for module, actions in permissions['function'].items(): + if module not in function_permissions: + function_permissions[module] = [] + function_permissions[module].extend(actions) + # 确保不重复 + function_permissions[module] = list(set(function_permissions[module])) + + # 合并数据权限 + if permissions.get('data'): + data_perms = permissions['data'] + + # 项目权限 + if data_perms.get('project_scope') == 'custom': + has_custom_project_scope = True + if data_permissions['project_scope'] == 'all': + data_permissions['project_scope'] = 'custom' + data_permissions['project_ids'] = data_perms.get('project_ids', []) + else: + # 合并项目ID列表 + data_permissions['project_ids'].extend(data_perms.get('project_ids', [])) + # 确保不重复 + data_permissions['project_ids'] = list(set(data_permissions['project_ids'])) + + # 环境权限 + if data_perms.get('environment_scope') == 'custom': + has_custom_environment_scope = True + if data_permissions['environment_scope'] == 'all': + data_permissions['environment_scope'] = 'custom' + data_permissions['environment_types'] = data_perms.get('environment_types', []) + else: + # 合并环境类型列表 + data_permissions['environment_types'].extend(data_perms.get('environment_types', [])) + # 确保不重复 + data_permissions['environment_types'] = list(set(data_permissions['environment_types'])) + + # 如果没有任何角色有自定义项目/环境范围，保持为'all' + if not has_custom_project_scope: + data_permissions['project_scope'] = 'all' + data_permissions['project_ids'] = [] + + if not has_custom_environment_scope: + data_permissions['environment_scope'] = 'all' + data_permissions['environment_types'] = [] + + return { + 'menu': list(menu_permissions), + 'function': function_permissions, + 'data': data_permissions + } + except Exception as e: + logger.error(f'获取用户权限失败: {str(e)}', exc_info=True) + # 返回默认的空权限 + return { + 'menu': [], + 'function': {}, + 'data': { + 'project_scope': 'all', + 'project_ids': [], + 'environment_scope': 'all', + 'environment_types': [] + } + } \ No newline at end of file diff --git a/backend/apps/utils/security.py b/backend/apps/utils/security.py new file mode 100644 index 0000000..8647e8d --- /dev/null +++ b/backend/apps/utils/security.py @@ -0,0 +1,214 @@ +import re +import hashlib +import logging +from datetime import datetime, timedelta +from django.utils import timezone +from ..models import SecurityConfig, LoginAttempt, User + +logger = logging.getLogger('apps') + +class SecurityValidator: + """安全验证工具类""" + + @staticmethod + def get_security_config(): + """获取安全配置""" + try: + config, created = SecurityConfig.objects.get_or_create( + id=1, + defaults={ + 'min_password_length': 8, + 'password_complexity': ['lowercase', 'number'], + 'session_timeout': 120, + 'max_login_attempts': 5, + 'lockout_duration': 30, + 'enable_2fa': False + } + ) + return config + except Exception as e: + logger.error(f'获取安全配置失败: {str(e)}') + # 返回默认配置 + return type('SecurityConfig', (), { + 'min_password_length': 8, + 'password_complexity': ['lowercase', 'number'], + 'session_timeout': 120, + 'max_login_attempts': 5, + 'lockout_duration': 30, + 'enable_2fa': False + })() + + @staticmethod + def validate_password(password): + """验证密码是否符合安全策略""" + config = SecurityValidator.get_security_config() + + # 检查密码长度 + if len(password) < config.min_password_length: + return False, f'密码长度不能少于{config.min_password_length}位' + + # 检查密码复杂度 + complexity_checks = { + 'uppercase': (r'[A-Z]', '大写字母'), + 'lowercase': (r'[a-z]', '小写字母'), + 'number': (r'[0-9]', '数字'), + 'special': (r'[!@#$%^&*(),.?":{}|<>]', '特殊字符') + } + + missing_requirements = [] + for requirement in config.password_complexity: + if requirement in complexity_checks: + pattern, description = complexity_checks[requirement] + if not re.search(pattern, password): + missing_requirements.append(description) + + if missing_requirements: + return False, f'密码必须包含: {", ".join(missing_requirements)}' + + return True, '密码验证通过' + + @staticmethod + def check_account_lockout(user, ip_address): + """检查账户是否被锁定""" + try: + # 首先检查用户表中的status字段 + if user.status == 0: + return False, '账户已被锁定，请联系管理员解锁' + + config = SecurityValidator.get_security_config() + + # 获取登录尝试记录 + try: + attempt = LoginAttempt.objects.get(user=user, ip_address=ip_address) + except LoginAttempt.DoesNotExist: + return True, '账户未被锁定' + + # 如果账户被锁定，检查是否已过期 + if attempt.locked_until and attempt.locked_until > timezone.now(): + remaining_time = attempt.locked_until - timezone.now() + minutes = int(remaining_time.total_seconds() / 60) + return False, f'账户因登录失败次数过多被临时锁定，请在{minutes}分钟后重试' + + # 如果临时锁定已过期，重置失败次数并解锁 + if attempt.locked_until and attempt.locked_until <= timezone.now(): + attempt.failed_attempts = 0 + attempt.locked_until = None + attempt.save() + + # 如果用户被系统锁定（status=0），检查是否需要自动解锁 + if user.status == 0: + # 暂时保持锁定状态，需要管理员手动解锁 + return False, '账户已被锁定，请联系管理员解锁' + + return True, '账户未被锁定' + + except Exception as e: + logger.error(f'检查账户锁定状态失败: {str(e)}') + return True, '锁定检查跳过' + + @staticmethod + def record_failed_login(user, ip_address): + """记录登录失败""" + try: + config = SecurityValidator.get_security_config() + + # 获取或创建登录尝试记录 + attempt, created = LoginAttempt.objects.get_or_create( + user=user, + ip_address=ip_address, + defaults={'failed_attempts': 0} + ) + + # 增加失败次数 + attempt.failed_attempts += 1 + attempt.last_attempt_time = timezone.now() + + # 如果达到最大失败次数，锁定账户 + if attempt.failed_attempts >= config.max_login_attempts: + attempt.locked_until = timezone.now() + timedelta(minutes=config.lockout_duration) + + # 同时将用户状态设置为锁定 + user.status = 0 + user.save() + + logger.warning(f'用户[{user.username}]账户因多次登录失败被锁定，IP: {ip_address}') + + attempt.save() + + return attempt.failed_attempts, config.max_login_attempts + + except Exception as e: + logger.error(f'记录登录失败失败: {str(e)}') + return 0, 5 + + @staticmethod + def record_successful_login(user, ip_address): + """记录登录成功，重置失败次数""" + try: + # 清除登录失败记录 + LoginAttempt.objects.filter( + user=user, + ip_address=ip_address + ).delete() + + except Exception as e: + logger.error(f'重置登录失败记录失败: {str(e)}') + + @staticmethod + def unlock_user_account(user): + """解锁用户账户（管理员操作）""" + try: + # 清除所有登录尝试记录 + LoginAttempt.objects.filter(user=user).delete() + + # 解锁用户状态 + if user.status == 0: + user.status = 1 + user.save() + logger.info(f'管理员解锁了用户[{user.username}]的账户') + return True, '账户解锁成功' + + return True, '账户状态正常' + + except Exception as e: + logger.error(f'解锁用户账户失败: {str(e)}') + return False, '解锁失败' + + @staticmethod + def lock_user_account(user): + """锁定用户账户（管理员操作）""" + try: + # 锁定用户状态 + if user.status == 1: + user.status = 0 + user.save() + logger.info(f'管理员锁定了用户[{user.username}]的账户') + return True, '账户锁定成功' + + return True, '账户已处于锁定状态' + + except Exception as e: + logger.error(f'锁定用户账户失败: {str(e)}') + return False, '锁定失败' + + @staticmethod + def is_session_expired(login_time): + """检查会话是否过期""" + try: + config = SecurityValidator.get_security_config() + + if not login_time: + return True + + # 计算会话过期时间 + session_expire_time = login_time + timedelta(minutes=config.session_timeout) + + return timezone.now() > session_expire_time + + except Exception as e: + logger.error(f'检查会话过期失败: {str(e)}') + return False + +def validate_password_strength(password): + """独立的密码强度验证函数，供其他模块使用""" + return SecurityValidator.validate_password(password) \ No newline at end of file diff --git a/backend/apps/views/build.py b/backend/apps/views/build.py new file mode 100644 index 0000000..d05854d --- /dev/null +++ b/backend/apps/views/build.py @@ -0,0 +1,924 @@ +import json +import uuid +import hashlib +import logging +import threading +import time +from datetime import datetime + +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.db import transaction +from django.db.models import Q, F +from ..models import BuildTask, BuildHistory, Project, Environment, GitlabTokenCredential, User, NotificationRobot +from ..utils.auth import jwt_auth_required +from ..utils.builder import Builder +from ..utils.permissions import get_user_permissions + +logger = logging.getLogger('apps') + +def generate_id(): + """生成唯一ID""" + return hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:32] + +def execute_build(task, build_number, commit_id, history): + """执行构建任务""" + try: + builder = Builder(task, build_number, commit_id, history) + builder.execute() + finally: + # 无论构建成功、失败或异常，都将构建状态重置为空闲 + from django.db import transaction + with transaction.atomic(): + # 重新获取任务对象，确保获取最新状态 + from ..models import BuildTask + BuildTask.objects.filter(task_id=task.task_id).update(building_status='idle') + logger.info(f"任务 [{task.task_id}] 构建状态已重置为空闲") + +@method_decorator(csrf_exempt, name='dispatch') +class BuildTaskView(View): + @method_decorator(jwt_auth_required) + def get(self, request, task_id=None): + """获取构建任务列表或单个任务详情""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + + # 检查用户是否有构建任务查看权限 + function_permissions = user_permissions.get('function', {}) + build_permissions = function_permissions.get('build_task', []) + + if 'view' not in build_permissions: + logger.warning(f'用户[{request.user_id}]没有构建任务查看权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看构建任务' + }, status=403) + + # 如果请求参数中包含get_robots=true，则返回通知机器人列表 + if request.GET.get('get_robots') == 'true': + robots = NotificationRobot.objects.all() + robot_list = [] + for robot in robots: + robot_list.append({ + 'robot_id': robot.robot_id, + 'type': robot.type, + 'name': robot.name, + }) + return JsonResponse({ + 'code': 200, + 'message': '获取通知机器人列表成功', + 'data': robot_list + }) + + # 如果提供了task_id，则返回单个任务详情 + if task_id: + try: + task = BuildTask.objects.select_related( + 'project', + 'environment', + 'git_token', + 'creator' + ).get(task_id=task_id) + + # 检查是否有权限查看该任务（项目权限和环境权限） + # 项目权限检查 + project_scope = data_permissions.get('project_scope', 'all') + if project_scope == 'custom': + permitted_project_ids = data_permissions.get('project_ids', []) + if task.project and task.project.project_id not in permitted_project_ids: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的项目[{task.project.project_id}]的构建任务') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该项目的构建任务' + }, status=403) + + # 环境权限检查 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_scope == 'custom': + permitted_environment_types = data_permissions.get('environment_types', []) + if task.environment and task.environment.type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的环境类型[{task.environment.type}]的构建任务') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该环境的构建任务' + }, status=403) + + # 获取最新的构建历史 + latest_build = BuildHistory.objects.filter(task=task).order_by('-build_number').first() + + # 获取通知机器人详情 + notification_robots = [] + if task.notification_channels: + robots = NotificationRobot.objects.filter(robot_id__in=task.notification_channels) + for robot in robots: + notification_robots.append({ + 'robot_id': robot.robot_id, + 'type': robot.type, + 'name': robot.name + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取任务详情成功', + 'data': { + 'task_id': task.task_id, + 'name': task.name, + 'project': { + 'project_id': task.project.project_id, + 'name': task.project.name, + 'repository': task.project.repository + } if task.project else None, + 'environment': { + 'environment_id': task.environment.environment_id, + 'name': task.environment.name, + 'type': task.environment.type + } if task.environment else None, + 'description': task.description, + 'requirement': task.requirement, + 'branch': task.branch, + 'git_token': { + 'credential_id': task.git_token.credential_id, + 'name': task.git_token.name + } if task.git_token else None, + 'stages': task.stages, + 'notification_channels': task.notification_channels, + 'notification_robots': notification_robots, + # 外部脚本库配置 + 'use_external_script': task.use_external_script, + 'external_script_repo_url': task.external_script_config.get('repo_url', '') if task.external_script_config else '', + 'external_script_directory': task.external_script_config.get('directory', '') if task.external_script_config else '', + 'external_script_branch': task.external_script_config.get('branch', '') if task.external_script_config else '', + 'external_script_token_id': task.external_script_config.get('token_id') if task.external_script_config else None, + 'status': task.status, + 'building_status': task.building_status, # 添加构建状态字段 + 'version': task.version, + 'last_build_number': task.last_build_number, + 'total_builds': task.total_builds, + 'success_builds': task.success_builds, + 'failure_builds': task.failure_builds, + 'last_build': { + 'id': latest_build.history_id, + 'number': latest_build.build_number, + 'status': latest_build.status, + 'time': latest_build.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'duration': '未完成' if latest_build.status in ['pending', 'running'] else str(latest_build.build_time.get('total_duration', 0)) + '秒' + } if latest_build else None, + 'creator': { + 'user_id': task.creator.user_id, + 'name': task.creator.name + } if task.creator else None, + 'create_time': task.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'update_time': task.update_time.strftime('%Y-%m-%d %H:%M:%S') + } + }) + except BuildTask.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '任务不存在' + }) + + # 获取查询参数 + project_id = request.GET.get('project_id') + environment_id = request.GET.get('environment_id') + name = request.GET.get('name') + + # 构建查询条件 + query = Q() + + # 应用项目权限过滤 + project_scope = data_permissions.get('project_scope', 'all') + if project_scope == 'custom': + permitted_project_ids = data_permissions.get('project_ids', []) + if not permitted_project_ids: + # 如果设置了自定义项目权限但列表为空，意味着没有权限查看任何项目 + logger.info(f'用户[{request.user_id}]没有权限查看任何项目的构建任务') + return JsonResponse({ + 'code': 200, + 'message': '获取任务列表成功', + 'data': [] + }) + + # 用户只能查看有权限的项目 + if project_id and project_id != 'all': + # 如果指定了项目，检查是否有该项目的权限 + if project_id not in permitted_project_ids: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的项目[{project_id}]的构建任务') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该项目的构建任务' + }, status=403) + query &= Q(project__project_id=project_id) + else: + # 如果没有指定项目或选择了全部，则限制为有权限的项目 + query &= Q(project__project_id__in=permitted_project_ids) + else: + # 如果有全部项目权限，并且指定了项目ID + if project_id and project_id != 'all': + query &= Q(project__project_id=project_id) + + # 应用环境权限过滤 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_scope == 'custom': + permitted_environment_types = data_permissions.get('environment_types', []) + if not permitted_environment_types: + # 如果设置了自定义环境权限但列表为空，意味着没有权限查看任何环境 + logger.info(f'用户[{request.user_id}]没有权限查看任何环境的构建任务') + return JsonResponse({ + 'code': 200, + 'message': '获取任务列表成功', + 'data': [] + }) + + if environment_id and environment_id != 'all': + # 如果指定了环境，需要检查是否在有权限的环境类型中 + try: + env = Environment.objects.get(environment_id=environment_id) + if env.type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的环境[{environment_id}]的构建任务') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该环境的构建任务' + }, status=403) + query &= Q(environment__environment_id=environment_id) + except Environment.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '环境不存在' + }, status=404) + else: + # 如果没有指定环境或选择了全部，则限制为有权限的环境类型 + query &= Q(environment__type__in=permitted_environment_types) + else: + # 如果有全部环境权限，并且指定了环境ID + if environment_id and environment_id != 'all': + query &= Q(environment__environment_id=environment_id) + + # 添加其他查询条件 + if name: + query &= Q(name__icontains=name) + + # 查询任务列表 + tasks = BuildTask.objects.select_related( + 'project', + 'environment', + 'creator' + ).filter(query) + + task_list = [] + for task in tasks: + # 获取最新的构建历史 + latest_build = BuildHistory.objects.filter(task=task).order_by('-build_number').first() + + task_list.append({ + 'task_id': task.task_id, + 'name': task.name, + 'project': { + 'project_id': task.project.project_id, + 'name': task.project.name + } if task.project else None, + 'environment': { + 'environment_id': task.environment.environment_id, + 'name': task.environment.name, + 'type': task.environment.type + } if task.environment else None, + 'description': task.description, + 'branch': task.branch, + 'status': task.status, + 'building_status': task.building_status, # 添加构建状态字段 + 'version': task.version, + 'last_build_number': task.last_build_number, + 'total_builds': task.total_builds, + 'success_builds': task.success_builds, + 'failure_builds': task.failure_builds, + 'last_build': { + 'id': latest_build.history_id, + 'number': latest_build.build_number, + 'status': latest_build.status, + 'time': latest_build.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'duration': '未完成' if latest_build.status in ['pending', 'running'] else str(latest_build.build_time.get('total_duration', 0)) + '秒' + } if latest_build else None, + 'creator': { + 'user_id': task.creator.user_id, + 'name': task.creator.name + } if task.creator else None, + 'create_time': task.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'update_time': task.update_time.strftime('%Y-%m-%d %H:%M:%S') + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取任务列表成功', + 'data': task_list + }) + except Exception as e: + logger.error(f'获取构建任务失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def post(self, request): + """创建构建任务""" + try: + with transaction.atomic(): + data = json.loads(request.body) + name = data.get('name') + project_id = data.get('project_id') + environment_id = data.get('environment_id') + description = data.get('description') + branch = data.get('branch', 'main') + git_token_id = data.get('git_token_id') + stages = data.get('stages', []) + notification_channels = data.get('notification_channels', []) + + # 外部脚本库配置 + use_external_script = data.get('use_external_script') + external_script_config = None + if 'use_external_script' in data: + if use_external_script: + repo_url = data.get('external_script_repo_url', '').strip() + directory = data.get('external_script_directory', '').strip() + external_script_branch = data.get('external_script_branch', '').strip() + token_id = data.get('external_script_token_id') + + # 验证外部脚本库必填字段 + if not repo_url: + return JsonResponse({ + 'code': 400, + 'message': '外部脚本库仓库地址不能为空' + }) + if not directory: + return JsonResponse({ + 'code': 400, + 'message': '外部脚本库存放目录不能为空' + }) + if not external_script_branch: + return JsonResponse({ + 'code': 400, + 'message': '外部脚本库分支名称不能为空' + }) + + external_script_config = { + 'repo_url': repo_url, + 'directory': directory, + 'branch': external_script_branch, + 'token_id': token_id + } + else: + external_script_config = {} + + # 验证必要字段 + if not all([name, project_id, environment_id]): + return JsonResponse({ + 'code': 400, + 'message': '任务名称、项目和环境不能为空' + }) + + # 验证通知机器人是否存在 + if notification_channels: + existing_robots = set(NotificationRobot.objects.filter( + robot_id__in=notification_channels + ).values_list('robot_id', flat=True)) + invalid_robots = set(notification_channels) - existing_robots + if invalid_robots: + return JsonResponse({ + 'code': 400, + 'message': f'以下机器人不存在: {", ".join(invalid_robots)}' + }) + + # 检查项目是否存在 + try: + project = Project.objects.get(project_id=project_id) + except Project.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '项目不存在' + }) + + # 检查环境是否存在 + try: + environment = Environment.objects.get(environment_id=environment_id) + except Environment.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '环境不存在' + }) + + # 检查GitLab Token凭证是否存在 + git_token = None + if git_token_id: + try: + git_token = GitlabTokenCredential.objects.get(credential_id=git_token_id) + except GitlabTokenCredential.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': 'GitLab Token凭证不存在' + }) + + # 创建构建任务 + creator = User.objects.get(user_id=request.user_id) + task = BuildTask.objects.create( + task_id=generate_id(), + name=name, + project=project, + environment=environment, + description=description, + branch=branch, + git_token=git_token, + stages=stages, + notification_channels=notification_channels, + use_external_script=use_external_script, + external_script_config=external_script_config, + creator=creator + ) + + return JsonResponse({ + 'code': 200, + 'message': '创建构建任务成功', + 'data': { + 'task_id': task.task_id, + 'name': task.name + } + }) + except Exception as e: + logger.error(f'创建构建任务失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """更新构建任务""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + function_permissions = user_permissions.get('function', {}) + build_permissions = function_permissions.get('build_task', []) + + with transaction.atomic(): + data = json.loads(request.body) + + task_id = data.get('task_id') + name = data.get('name') + project_id = data.get('project_id') + environment_id = data.get('environment_id') + description = data.get('description') + branch = data.get('branch') + git_token_id = data.get('git_token_id') + stages = data.get('stages') + notification_channels = data.get('notification_channels') + status = data.get('status') + + # 外部脚本库配置 + use_external_script = data.get('use_external_script') + external_script_config = None + if 'use_external_script' in data: + if use_external_script: + repo_url = data.get('external_script_repo_url', '').strip() + directory = data.get('external_script_directory', '').strip() + external_script_branch = data.get('external_script_branch', '').strip() + token_id = data.get('external_script_token_id') + + # 验证外部脚本库必填字段 + if not repo_url: + return JsonResponse({ + 'code': 400, + 'message': '外部脚本库仓库地址不能为空' + }) + if not directory: + return JsonResponse({ + 'code': 400, + 'message': '外部脚本库存放目录不能为空' + }) + if not external_script_branch: + return JsonResponse({ + 'code': 400, + 'message': '外部脚本库分支名称不能为空' + }) + + external_script_config = { + 'repo_url': repo_url, + 'directory': directory, + 'branch': external_script_branch, + 'token_id': token_id + } + else: + external_script_config = {} + + if not task_id: + return JsonResponse({ + 'code': 400, + 'message': '任务ID不能为空' + }) + + try: + task = BuildTask.objects.get(task_id=task_id) + except BuildTask.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '任务不存在' + }) + + # 如果只修改状态，需要检查是否有禁用权限 + if status and len(data) == 2 and 'task_id' in data and 'status' in data: + if 'disable' not in build_permissions: + logger.warning(f'用户[{request.user_id}]没有禁用/启用任务权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限禁用/启用任务' + }, status=403) + else: + # 否则检查是否有编辑权限 + if 'edit' not in build_permissions: + logger.warning(f'用户[{request.user_id}]没有编辑任务权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限编辑任务' + }, status=403) + + # 项目权限检查 + project_scope = data_permissions.get('project_scope', 'all') + if project_id and project_scope == 'custom': + permitted_project_ids = data_permissions.get('project_ids', []) + if project_id not in permitted_project_ids: + logger.warning(f'用户[{request.user_id}]尝试编辑无权限的项目[{project_id}]的构建任务') + return JsonResponse({ + 'code': 403, + 'message': '没有权限编辑该项目的构建任务' + }, status=403) + + # 环境权限检查 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_id and environment_scope == 'custom': + try: + env = Environment.objects.get(environment_id=environment_id) + permitted_environment_types = data_permissions.get('environment_types', []) + if env.type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试编辑无权限的环境类型[{env.type}]的构建任务') + return JsonResponse({ + 'code': 403, + 'message': '没有权限编辑该环境的构建任务' + }, status=403) + except Environment.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '环境不存在' + }) + + # 更新项目关联 + if project_id: + try: + project = Project.objects.get(project_id=project_id) + task.project = project + except Project.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '项目不存在' + }) + + # 更新环境关联 + if environment_id: + try: + environment = Environment.objects.get(environment_id=environment_id) + task.environment = environment + except Environment.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '环境不存在' + }) + + # 更新GitLab Token凭证关联 + if 'git_token_id' in data: + if git_token_id: + try: + git_token = GitlabTokenCredential.objects.get(credential_id=git_token_id) + task.git_token = git_token + except GitlabTokenCredential.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': 'GitLab Token凭证不存在' + }) + else: + task.git_token = None + + # 更新其他字段 + if 'name' in data: + task.name = name + if 'description' in data: + task.description = description + if 'branch' in data: + task.branch = branch + if 'stages' in data: + task.stages = stages + if 'notification_channels' in data: + # 验证通知机器人是否存在 + existing_robots = set(NotificationRobot.objects.filter( + robot_id__in=notification_channels + ).values_list('robot_id', flat=True)) + invalid_robots = set(notification_channels) - existing_robots + if invalid_robots: + return JsonResponse({ + 'code': 400, + 'message': f'以下机器人不存在: {", ".join(invalid_robots)}' + }) + task.notification_channels = notification_channels + if 'status' in data: + task.status = status + + # 更新外部脚本库配置 + if 'use_external_script' in data: + task.use_external_script = use_external_script + task.external_script_config = external_script_config + + task.save() + + return JsonResponse({ + 'code': 200, + 'message': '更新构建任务成功' + }) + except Exception as e: + logger.error(f'更新构建任务失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def delete(self, request): + """删除构建任务""" + try: + with transaction.atomic(): + data = json.loads(request.body) + task_id = data.get('task_id') + + if not task_id: + return JsonResponse({ + 'code': 400, + 'message': '任务ID不能为空' + }) + + try: + task = BuildTask.objects.get(task_id=task_id) + task.delete() + return JsonResponse({ + 'code': 200, + 'message': '删除构建任务成功' + }) + except BuildTask.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '任务不存在' + }) + except Exception as e: + logger.error(f'删除构建任务失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + +@method_decorator(csrf_exempt, name='dispatch') +class BuildExecuteView(View): + @method_decorator(jwt_auth_required) + def post(self, request): + """执行构建""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + + # 检查用户是否有执行构建权限 + function_permissions = user_permissions.get('function', {}) + build_permissions = function_permissions.get('build_task', []) + + if 'execute' not in build_permissions: + logger.warning(f'用户[{request.user_id}]没有执行构建权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限执行构建任务' + }, status=403) + + data = json.loads(request.body) + task_id = data.get('task_id') + branch = data.get('branch') # 获取用户选择的分支 + commit_id = data.get('commit_id') + version = data.get('version') + requirement = data.get('requirement') + + if not task_id: + return JsonResponse({ + 'code': 400, + 'message': '任务ID不能为空' + }) + + try: + task = BuildTask.objects.select_related('project', 'environment').get(task_id=task_id) + except BuildTask.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '任务不存在' + }) + + # 检查是否有权限执行该任务 + # 项目权限检查 + project_scope = data_permissions.get('project_scope', 'all') + if project_scope == 'custom': + permitted_project_ids = data_permissions.get('project_ids', []) + if task.project and task.project.project_id not in permitted_project_ids: + logger.warning(f'用户[{request.user_id}]尝试执行无权限的项目[{task.project.project_id}]的构建任务') + return JsonResponse({ + 'code': 403, + 'message': '没有权限执行该项目的构建任务' + }, status=403) + + # 环境权限检查 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_scope == 'custom': + permitted_environment_types = data_permissions.get('environment_types', []) + if task.environment and task.environment.type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试执行无权限的环境类型[{task.environment.type}]的构建任务') + return JsonResponse({ + 'code': 403, + 'message': '没有权限执行该环境的构建任务' + }, status=403) + + # 根据环境类型检查必要参数 + env_type = task.environment.type if task.environment else None + + if env_type in ['development', 'testing']: + # 开发环境和测试环境需要分支和commit_id + if not branch: + return JsonResponse({ + 'code': 400, + 'message': '分支不能为空' + }) + + if not commit_id: + return JsonResponse({ + 'code': 400, + 'message': 'Commit ID不能为空' + }) + elif env_type in ['staging', 'production']: + if not version: + return JsonResponse({ + 'code': 400, + 'message': '版本号不能为空' + }) + parts = version.split('_') + if len(parts) == 2 and len(parts[1]) >= 8: + commit_id = parts[1] + else: + return JsonResponse({ + 'code': 400, + 'message': '版本号格式不正确，应为：YYYYMMDDHHmmSS_commitId' + }) + + if not requirement: + return JsonResponse({ + 'code': 400, + 'message': '构建需求描述不能为空' + }) + + # 检查任务的构建状态 + if task.building_status == 'building': + return JsonResponse({ + 'code': 400, + 'message': '当前任务正在构建中，请等待构建完成后再试' + }) + + running_build = BuildHistory.objects.filter( + task_id=task_id, + status__in=['pending', 'running'] + ).exists() + + if running_build: + # 如果有正在进行的构建，但building_status不是building，则修正状态 + BuildTask.objects.filter(task_id=task_id).update(building_status='building') + return JsonResponse({ + 'code': 400, + 'message': '当前任务有正在进行的构建，请等待构建完成后再试' + }) + + if task.status == 'disabled': + return JsonResponse({ + 'code': 400, + 'message': '任务已禁用' + }) + + # 生成构建号 + build_number = task.last_build_number + 1 + + # 创建构建历史记录 + history = BuildHistory.objects.create( + history_id=generate_id(), + task=task, + build_number=build_number, + branch=branch if branch else '', # 对于预发布和生产环境，分支可能为空 + commit_id=commit_id, + version=version if version else None, # 对于预发布和生产环境，使用传入的版本号 + status='pending', # 初始状态为等待中 + requirement=requirement, + operator=User.objects.get(user_id=request.user_id) # 记录构建人 + ) + + # 更新任务状态、构建号和构建状态 + BuildTask.objects.filter(task_id=task_id).update( + last_build_number=build_number, + total_builds=F('total_builds') + 1, + building_status='building' # 设置为构建中状态 + ) + + # 在新线程中执行构建 + build_thread = threading.Thread( + target=execute_build, + args=(task, build_number, commit_id, history) + ) + build_thread.start() + + return JsonResponse({ + 'code': 200, + 'message': '开始构建', + 'data': { + 'build_number': build_number, + 'history_id': history.history_id + } + }) + except Exception as e: + logger.error(f'执行构建失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """停止构建""" + try: + data = json.loads(request.body) + history_id = data.get('history_id') + + if not history_id: + return JsonResponse({ + 'code': 400, + 'message': '历史ID不能为空' + }) + + try: + history = BuildHistory.objects.get(history_id=history_id) + except BuildHistory.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '构建历史不存在' + }) + + # 只有进行中的构建可以停止 + if history.status not in ['pending', 'running']: + return JsonResponse({ + 'code': 400, + 'message': '只能停止进行中的构建' + }) + + # 更新构建状态为terminated + history.status = 'terminated' + + # 如果构建日志存在，追加终止消息 + if history.build_log: + history.build_log += "\n[系统] 构建被手动终止\n" + else: + history.build_log = "[系统] 构建被手动终止\n" + + # 更新构建时间 + if not history.build_time: + history.build_time = {} + + if 'start_time' in history.build_time and 'total_duration' not in history.build_time: + # 计算从开始到现在的持续时间 + start_time = datetime.strptime(history.build_time['start_time'], '%Y-%m-%d %H:%M:%S') + duration = int((datetime.now() - start_time).total_seconds()) + history.build_time['total_duration'] = str(duration) + history.build_time['end_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + history.save() + + # 更新任务统计信息和构建状态 + BuildTask.objects.filter(task_id=history.task.task_id).update( + building_status='idle' # 重置构建状态为空闲 + ) + + return JsonResponse({ + 'code': 200, + 'message': '构建已终止' + }) + + except Exception as e: + logger.error(f'停止构建失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/build_history.py b/backend/apps/views/build_history.py new file mode 100644 index 0000000..b6434a8 --- /dev/null +++ b/backend/apps/views/build_history.py @@ -0,0 +1,549 @@ +import json +import logging +from datetime import datetime, timedelta +from django.http import JsonResponse, HttpResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.db.models import Q +from ..models import BuildHistory, BuildTask, Project, Environment +from ..utils.auth import jwt_auth_required +from ..utils.permissions import get_user_permissions + +logger = logging.getLogger('apps') + +@method_decorator(csrf_exempt, name='dispatch') +class BuildHistoryView(View): + def _get_stage_status_from_log(self, log: str, stage_name: str, overall_status: str = None) -> str: + """从日志中获取指定阶段的状态""" + if not log: + return 'failed' + + # 处理特殊阶段：Git Clone + if stage_name == 'Git Clone': + # Git Clone 阶段使用 [Git Clone] 格式 + if '[Git Clone]' not in log: + return 'failed' + + # 检查是否有完成标记 + if '[Git Clone] 代码克隆完成' in log: + return 'success' + elif overall_status == 'terminated': + return 'terminated' + else: + return 'failed' + + # 处理普通构建阶段 + stage_start_pattern = f'[Build Stages] 开始执行阶段: {stage_name}' + stage_complete_pattern = f'[Build Stages] 阶段 {stage_name} 执行完成' + + # 如果整体构建状态是terminated，检查阶段是否有开始执行 + if overall_status == 'terminated': + if stage_start_pattern in log: + # 检查阶段是否完成 + if stage_complete_pattern in log: + return 'success' + else: + return 'terminated' + else: + # 没有该阶段的日志，说明还没开始执行就被终止了 + return 'terminated' + + if stage_start_pattern not in log: + return 'failed' # 阶段未开始执行 + + # 检查阶段是否完成 + if stage_complete_pattern in log: + return 'success' # 阶段成功完成 + else: + return 'failed' # 阶段开始了但没有完成 + + @method_decorator(jwt_auth_required) + def get(self, request): + """获取构建历史列表""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + + # 检查用户是否有构建历史查看权限 + function_permissions = user_permissions.get('function', {}) + build_history_permissions = function_permissions.get('build_history', []) + + if 'view' not in build_history_permissions: + logger.warning(f'用户[{request.user_id}]没有构建历史查看权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看构建历史' + }, status=403) + + # 获取查询参数 + project_id = request.GET.get('project_id') + environment_id = request.GET.get('environment_id') + task_id = request.GET.get('task_id') # 添加task_id参数 + task_name = request.GET.get('task_name') + page = int(request.GET.get('page', 1)) + page_size = int(request.GET.get('page_size', 10)) + + # 构建查询条件 + query = Q() + + # 应用项目权限过滤 + project_scope = data_permissions.get('project_scope', 'all') + if project_scope == 'custom': + permitted_project_ids = data_permissions.get('project_ids', []) + if not permitted_project_ids: + logger.info(f'用户[{request.user_id}]没有权限查看任何项目的构建历史') + return JsonResponse({ + 'code': 200, + 'message': '获取构建历史列表成功', + 'data': [], + 'total': 0, + 'page': page, + 'page_size': page_size + }) + + # 用户只能查看有权限的项目 + if project_id and project_id != 'all': + # 如果指定了项目，检查是否有该项目的权限 + if project_id not in permitted_project_ids: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的项目[{project_id}]的构建历史') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该项目的构建历史' + }, status=403) + query &= Q(task__project__project_id=project_id) + else: + query &= Q(task__project__project_id__in=permitted_project_ids) + else: + # 如果有全部项目权限，并且指定了项目ID + if project_id and project_id != 'all': + query &= Q(task__project__project_id=project_id) + + # 应用环境权限过滤 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_scope == 'custom': + permitted_environment_types = data_permissions.get('environment_types', []) + if not permitted_environment_types: + # 如果设置了自定义环境权限但列表为空，意味着没有权限查看任何环境 + logger.info(f'用户[{request.user_id}]没有权限查看任何环境的构建历史') + return JsonResponse({ + 'code': 200, + 'message': '获取构建历史列表成功', + 'data': [], + 'total': 0, + 'page': page, + 'page_size': page_size + }) + + if environment_id and environment_id != 'all': + # 如果指定了环境，需要检查是否在有权限的环境类型中 + try: + env = Environment.objects.get(environment_id=environment_id) + if env.type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的环境[{environment_id}]的构建历史') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该环境的构建历史' + }, status=403) + query &= Q(task__environment__environment_id=environment_id) + except Environment.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '环境不存在' + }, status=404) + else: + # 如果没有指定环境或选择了全部，则限制为有权限的环境类型 + query &= Q(task__environment__type__in=permitted_environment_types) + else: + # 如果有全部环境权限，并且指定了环境ID + if environment_id and environment_id != 'all': + query &= Q(task__environment__environment_id=environment_id) + + # 添加其他查询条件 + if task_id: + query &= Q(task__task_id=task_id) + if task_name: + query &= Q(task__name__icontains=task_name) + + # 查询构建历史 + histories = BuildHistory.objects.select_related( + 'task', + 'task__project', + 'task__environment', + 'operator' + ).filter(query).order_by('-create_time') + + # 计算总数 + total = histories.count() + + # 分页 + start = (page - 1) * page_size + end = start + page_size + histories = histories[start:end] + + # 构建返回数据 + history_list = [] + for history in histories: + # 计算构建耗时 + duration = '未完成' + if history.build_time and 'total_duration' in history.build_time: + duration_seconds = int(history.build_time['total_duration']) + if duration_seconds < 60: + duration = f"{duration_seconds}秒" + else: + minutes = duration_seconds // 60 + seconds = duration_seconds % 60 + duration = f"{minutes}分{seconds}秒" + + # 处理构建阶段信息 + stages = [] + + # 添加 Git Clone 阶段 + git_clone_stage = next( + (t for t in history.build_time.get('stages_time', []) if t['name'] == 'Git Clone'), + None + ) if history.build_time else None + + if git_clone_stage: + git_clone_status = self._get_stage_status_from_log(history.build_log, 'Git Clone', history.status) + stages.append({ + 'name': 'Git Clone', + 'status': git_clone_status, + 'startTime': git_clone_stage['start_time'], + 'duration': git_clone_stage['duration'] + '秒' + }) + + # 添加其他阶段 + for stage in history.stages: + stage_time = next( + (t for t in history.build_time.get('stages_time', []) if t['name'] == stage['name']), + None + ) if history.build_time else None + + stage_status = self._get_stage_status_from_log(history.build_log, stage['name'], history.status) + stages.append({ + 'name': stage['name'], + 'status': stage_status, + 'startTime': stage_time['start_time'] if stage_time else None, + 'duration': stage_time['duration'] + '秒' if stage_time else '未知' + }) + + # 检查是否有回滚权限 + can_rollback = history.status == 'success' + + history_list.append({ + 'id': history.history_id, + 'build_number': history.build_number, + 'status': history.status, + 'branch': history.branch, + 'commit': history.commit_id[:8] if history.commit_id else None, + 'version': history.version, + 'environment': history.task.environment.name if history.task.environment else None, + 'startTime': history.build_time.get('start_time') if history.build_time else history.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'duration': duration, + 'operator': history.operator.name if history.operator else None, + 'requirement': history.requirement, + 'stages': stages, + 'canRollback': can_rollback, + 'task': { + 'id': history.task.task_id, + 'name': history.task.name, + 'description': history.task.description + } + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取构建历史列表成功', + 'data': history_list, + 'total': total, + 'page': page, + 'page_size': page_size + }) + + except Exception as e: + logger.error(f'获取构建历史列表失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def post(self, request): + """回滚到指定版本""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + function_permissions = user_permissions.get('function', {}) + build_history_permissions = function_permissions.get('build_history', []) + + # 检查是否有回滚权限 + if 'rollback' not in build_history_permissions: + logger.warning(f'用户[{request.user_id}]没有构建历史回滚权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限执行回滚操作' + }, status=403) + + data = json.loads(request.body) + history_id = data.get('history_id') + + if not history_id: + return JsonResponse({ + 'code': 400, + 'message': '历史ID不能为空' + }) + + try: + history = BuildHistory.objects.select_related('task', 'task__project', 'task__environment').get(history_id=history_id) + except BuildHistory.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '构建历史不存在' + }) + + # 检查项目和环境权限 + data_permissions = user_permissions.get('data', {}) + + # 项目权限检查 + project_scope = data_permissions.get('project_scope', 'all') + if project_scope == 'custom': + permitted_project_ids = data_permissions.get('project_ids', []) + if history.task.project.project_id not in permitted_project_ids: + logger.warning(f'用户[{request.user_id}]尝试回滚无权限的项目[{history.task.project.project_id}]的构建') + return JsonResponse({ + 'code': 403, + 'message': '没有权限回滚该项目的构建' + }, status=403) + + # 环境权限检查 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_scope == 'custom': + permitted_environment_types = data_permissions.get('environment_types', []) + if history.task.environment.type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试回滚无权限的环境类型[{history.task.environment.type}]的构建') + return JsonResponse({ + 'code': 403, + 'message': '没有权限回滚该环境的构建' + }, status=403) + + if history.status != 'success': + return JsonResponse({ + 'code': 400, + 'message': '只能回滚到构建成功的版本' + }) + + # TODO: 实现回滚逻辑 + + return JsonResponse({ + 'code': 200, + 'message': '开始回滚', + 'data': { + 'version': history.version + } + }) + + except Exception as e: + logger.error(f'回滚失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + +@method_decorator(csrf_exempt, name='dispatch') +class BuildLogView(View): + @method_decorator(jwt_auth_required) + def get(self, request, history_id): + """获取构建日志""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + + # 检查用户是否有构建历史查看日志权限 + function_permissions = user_permissions.get('function', {}) + build_task_permissions = function_permissions.get('build_task', []) + build_history_permissions = function_permissions.get('build_history', []) + + has_log_permission = 'view_log' in build_task_permissions or 'view_log' in build_history_permissions + + if not has_log_permission: + logger.warning(f'用户[{request.user_id}]没有构建历史查看日志权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看构建日志' + }, status=403) + + try: + history = BuildHistory.objects.select_related('task', 'task__project', 'task__environment').get(history_id=history_id) + except BuildHistory.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '构建历史不存在' + }) + + # 项目权限检查 + project_scope = data_permissions.get('project_scope', 'all') + if project_scope == 'custom': + permitted_project_ids = data_permissions.get('project_ids', []) + if history.task.project and history.task.project.project_id not in permitted_project_ids: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的项目[{history.task.project.project_id}]的构建日志') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该项目的构建日志' + }, status=403) + + # 环境权限检查 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_scope == 'custom': + permitted_environment_types = data_permissions.get('environment_types', []) + if history.task.environment and history.task.environment.type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的环境类型[{history.task.environment.type}]的构建日志') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该环境的构建日志' + }, status=403) + + # 检查是否为下载请求 + is_download = request.GET.get('download') == 'true' + if is_download: + # 生成日志文件名 + filename = f"build_log_{history.task.name}_{history.build_number}.txt" + + # 准备日志内容 + log_content = history.build_log or '暂无日志' + + # 创建响应对象 + response = HttpResponse(log_content, content_type='text/plain') + response['Content-Disposition'] = f'attachment; filename="{filename}"' + return response + + return JsonResponse({ + 'code': 200, + 'message': '获取构建日志成功', + 'data': { + 'log': history.build_log or '暂无日志' + } + }) + + except Exception as e: + logger.error(f'获取构建日志失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + +@method_decorator(csrf_exempt, name='dispatch') +class BuildStageLogView(View): + @method_decorator(jwt_auth_required) + def get(self, request, history_id, stage_name): + """获取构建阶段日志""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + + # 检查用户是否有构建历史查看日志权限 + function_permissions = user_permissions.get('function', {}) + build_task_permissions = function_permissions.get('build_task', []) + build_history_permissions = function_permissions.get('build_history', []) + + # 只要有任何一方的view_log权限即可 + has_log_permission = 'view_log' in build_task_permissions or 'view_log' in build_history_permissions + + if not has_log_permission: + logger.warning(f'用户[{request.user_id}]没有构建历史查看日志权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看构建日志' + }, status=403) + + try: + history = BuildHistory.objects.select_related('task', 'task__project', 'task__environment').get(history_id=history_id) + except BuildHistory.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '构建历史不存在' + }) + + # 项目权限检查 + project_scope = data_permissions.get('project_scope', 'all') + if project_scope == 'custom': + permitted_project_ids = data_permissions.get('project_ids', []) + if history.task.project and history.task.project.project_id not in permitted_project_ids: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的项目[{history.task.project.project_id}]的构建阶段日志') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该项目的构建日志' + }, status=403) + + # 环境权限检查 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_scope == 'custom': + permitted_environment_types = data_permissions.get('environment_types', []) + if history.task.environment and history.task.environment.type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的环境类型[{history.task.environment.type}]的构建阶段日志') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该环境的构建日志' + }, status=403) + + # 在完整日志中查找指定阶段的日志 + if not history.build_log: + return JsonResponse({ + 'code': 200, + 'message': '获取阶段日志成功', + 'data': { + 'log': '暂无日志' + } + }) + + # 适配Jenkins风格日志格式的阶段日志解析 + stage_logs = [] + lines = history.build_log.split('\n') + in_stage = False + + # 处理特殊阶段：Git Clone + if stage_name == 'Git Clone': + # Git Clone 阶段使用 [Git Clone] 格式 + for line in lines: + if '[Git Clone]' in line: + stage_logs.append(line) + else: + # 普通构建阶段使用 [Build Stages] 格式 + stage_start_pattern = f'[Build Stages] 开始执行阶段: {stage_name}' + stage_complete_pattern = f'[Build Stages] 阶段 {stage_name} 执行完成' + + for line in lines: + if stage_start_pattern in line: + in_stage = True + stage_logs.append(line) + elif in_stage and stage_complete_pattern in line: + stage_logs.append(line) + break # 阶段结束 + elif in_stage: + # 在阶段执行期间的所有日志都属于该阶段 + # 排除其他阶段的开始标记 + if '[Build Stages] 开始执行阶段:' not in line: + stage_logs.append(line) + else: + # 遇到其他阶段开始，当前阶段结束 + break + + return JsonResponse({ + 'code': 200, + 'message': '获取阶段日志成功', + 'data': { + 'log': '\n'.join(stage_logs) if stage_logs else '暂无该阶段日志' + } + }) + + except Exception as e: + logger.error(f'获取阶段日志失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/build_sse.py b/backend/apps/views/build_sse.py new file mode 100644 index 0000000..494e563 --- /dev/null +++ b/backend/apps/views/build_sse.py @@ -0,0 +1,273 @@ +import json +import time +import logging +import uuid +import threading +import queue +import asyncio +from django.http import StreamingHttpResponse, JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from asgiref.sync import sync_to_async +from ..utils.auth import jwt_auth_required +from ..utils.log_stream import log_stream_manager +from ..models import BuildHistory, UserToken + +logger = logging.getLogger('apps') + +@method_decorator(csrf_exempt, name='dispatch') +class BuildLogSSEView(View): + """构建日志SSE流视图 - 异步实现以支持ASGI环境""" + + def options(self, request, task_id, build_number): + """处理CORS预检请求""" + response = JsonResponse({}) + response['Access-Control-Allow-Origin'] = '*' + response['Access-Control-Allow-Methods'] = 'GET, OPTIONS' + response['Access-Control-Allow-Headers'] = 'Cache-Control, Authorization' + response['Access-Control-Max-Age'] = '86400' # 24小时 + return response + + def get(self, request, task_id, build_number): + """获取构建日志SSE流""" + try: + # 从URL参数获取token进行认证 + token = request.GET.get('token') + if not token: + return StreamingHttpResponse( + self._create_error_stream("缺少认证token"), + content_type='text/event-stream' + ) + + # 验证token + try: + user_info = self._verify_jwt_token(token) + if not user_info: + return StreamingHttpResponse( + self._create_error_stream("无效的认证token"), + content_type='text/event-stream' + ) + except Exception as e: + logger.error(f"Token验证失败: {str(e)}", exc_info=True) + return StreamingHttpResponse( + self._create_error_stream("认证失败"), + content_type='text/event-stream' + ) + + # 验证构建历史记录是否存在 + try: + history = BuildHistory.objects.get( + task__task_id=task_id, + build_number=int(build_number) + ) + except BuildHistory.DoesNotExist: + return StreamingHttpResponse( + self._create_error_stream("构建记录不存在"), + content_type='text/event-stream' + ) + except ValueError: + return StreamingHttpResponse( + self._create_error_stream("无效的构建号"), + content_type='text/event-stream' + ) + + # 创建异步SSE流 + response = StreamingHttpResponse( + self._build_log_stream_async(task_id, int(build_number), history), + content_type='text/event-stream' + ) + + # 设置SSE相关的HTTP头 + response['Cache-Control'] = 'no-cache, no-store, must-revalidate' + response['Pragma'] = 'no-cache' + response['Expires'] = '0' + response['Access-Control-Allow-Origin'] = '*' + response['Access-Control-Allow-Headers'] = 'Cache-Control' + response['Access-Control-Allow-Methods'] = 'GET' + response['X-Accel-Buffering'] = 'no' # 禁用nginx缓冲 + + return response + + except Exception as e: + logger.error(f"创建SSE流失败: {str(e)}", exc_info=True) + return StreamingHttpResponse( + self._create_error_stream(f"服务器错误: {str(e)}"), + content_type='text/event-stream' + ) + + def _verify_jwt_token(self, token): + """验证JWT token""" + try: + import jwt + from django.conf import settings + + # 查询用户token + user_token = UserToken.objects.filter(token=token).first() + + if not user_token: + logger.warning("Token不存在于数据库中") + return None + + # 验证JWT token + try: + payload = jwt.decode(token, settings.SECRET_KEY, algorithms=['HS256']) + return { + 'user_id': payload.get('user_id'), + 'username': payload.get('username') + } + except jwt.ExpiredSignatureError: + logger.warning("Token已过期") + return None + except jwt.InvalidTokenError as e: + logger.warning(f"Token无效: {str(e)}") + return None + + except Exception as e: + logger.error(f"Token验证过程发生错误: {str(e)}", exc_info=True) + return None + + async def _build_log_stream_async(self, task_id, build_number, history): + """异步生成构建日志流""" + # 生成唯一的客户端ID + client_id = str(uuid.uuid4()) + + try: + # 发送连接建立消息 + yield self._format_sse_message({ + 'type': 'connection_established', + 'message': '连接成功，开始接收构建日志...' + }) + + # 如果构建已完成，发送完成消息并结束 + if history.status in ['success', 'failed', 'terminated']: + yield self._format_sse_message({ + 'type': 'build_complete', + 'status': history.status, + 'message': f'构建已完成，状态: {history.status}。请使用历史日志API获取完整日志。' + }) + return + + # 对于正在进行的构建，使用实时日志流 + heartbeat_counter = 0 + + # 获取异步日志流 + async for log_msg in self._async_log_stream(task_id, build_number, client_id): + if log_msg is None: + # 心跳包 + heartbeat_counter += 1 + if heartbeat_counter >= 30: # 每30秒发送一次心跳 + yield self._format_sse_message({ + 'type': 'heartbeat', + 'timestamp': int(time.time()) + }, event_type='heartbeat') + heartbeat_counter = 0 + continue + + # 重置心跳计数器 + heartbeat_counter = 0 + + # 检查是否是构建完成消息 + if log_msg.message.startswith('BUILD_COMPLETE:'): + status = log_msg.message.split(':', 1)[1] + yield self._format_sse_message({ + 'type': 'build_complete', + 'status': status, + 'message': f'构建已完成，状态: {status}' + }) + break + else: + # 普通日志消息 + yield self._format_sse_message({ + 'type': 'build_log', + 'message': log_msg.message + }) + + except Exception as e: + logger.error(f"生成构建日志流时发生错误: {str(e)}", exc_info=True) + yield self._format_sse_message({ + 'type': 'error', + 'message': f'日志流发生错误: {str(e)}' + }) + + async def _async_log_stream(self, task_id, build_number, client_id): + """异步日志流生成器 - 改进版本""" + try: + async_queue = asyncio.Queue(maxsize=1000) # 限制队列大小 + stop_event = asyncio.Event() + loop = asyncio.get_running_loop() + + def sync_log_reader(): + """在单独线程中读取同步日志流""" + try: + for log_msg in log_stream_manager.get_log_stream(task_id, build_number, client_id): + if stop_event.is_set(): + break + + # 使用线程安全的方式添加到异步队列 + try: + asyncio.run_coroutine_threadsafe( + async_queue.put(log_msg), loop + ).result(timeout=0.1) + except Exception as queue_error: + logger.debug(f"队列添加失败，可能客户端已断开: {queue_error}") + break + + # 发送结束信号 + try: + asyncio.run_coroutine_threadsafe( + async_queue.put(StopAsyncIteration), loop + ).result(timeout=0.1) + except Exception: + pass # 忽略结束信号发送失败 + + except Exception as e: + logger.error(f"同步日志读取器出错: {str(e)}", exc_info=True) + try: + asyncio.run_coroutine_threadsafe( + async_queue.put(StopAsyncIteration), loop + ).result(timeout=0.1) + except Exception: + pass + + # 在线程池中启动同步日志读取器 + thread = threading.Thread(target=sync_log_reader, daemon=True) + thread.start() + + try: + while True: + try: + # 异步等待日志消息，使用较短的超时以提供更好的响应性 + log_msg = await asyncio.wait_for(async_queue.get(), timeout=1.0) + + if log_msg is StopAsyncIteration: + break + + yield log_msg + + except asyncio.TimeoutError: + # 超时，发送心跳 + yield None + + finally: + # 设置停止事件，清理资源 + stop_event.set() + if thread.is_alive(): + thread.join(timeout=2.0) # 增加join超时时间 + + except Exception as e: + logger.error(f"异步日志流错误: {str(e)}", exc_info=True) + yield None # 确保生成器正常结束 + + def _create_error_stream(self, error_message): + """创建错误流""" + yield self._format_sse_message({ + 'type': 'error', + 'message': error_message + }) + + def _format_sse_message(self, data, event_type='message'): + """格式化SSE消息""" + message = f"event: {event_type}\n" + message += f"data: {json.dumps(data, ensure_ascii=False)}\n\n" + return message \ No newline at end of file diff --git a/backend/apps/views/credentials.py b/backend/apps/views/credentials.py new file mode 100644 index 0000000..00d1a90 --- /dev/null +++ b/backend/apps/views/credentials.py @@ -0,0 +1,1044 @@ +import json +import uuid +import hashlib +import logging +import os +import stat +import subprocess +import tempfile +import yaml +import shutil +from datetime import datetime +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.contrib.auth.hashers import make_password, check_password # check_password +from ..models import ( + GitlabTokenCredential, + SSHKeyCredential, + KubeconfigCredential, + User +) +from ..utils.auth import jwt_auth_required + +logger = logging.getLogger('apps') + +def generate_id(): + """生成唯一ID""" + return hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:32] + +def encrypt_sensitive_data(data, credential_type=None): + if not data: + return None + if credential_type in ['gitlab_token', 'ssh_key']: + return data + return make_password(data) + +CREDENTIAL_MODELS = { + 'gitlab_token': GitlabTokenCredential, + 'ssh_key': SSHKeyCredential, + 'kubeconfig': KubeconfigCredential, +} + +class SSHKeyManager: + """SSH密钥管理器 - 支持直接ssh user@host方式""" + + def __init__(self): + self.ssh_dir = os.path.expanduser('~/.ssh') + self.ensure_ssh_dir() + + def ensure_ssh_dir(self): + """确保SSH目录存在并设置正确权限""" + if not os.path.exists(self.ssh_dir): + os.makedirs(self.ssh_dir, mode=0o700) + else: + os.chmod(self.ssh_dir, 0o700) + + def deploy_ssh_key(self, credential_id, private_key, passphrase=None): + """部署SSH密钥到容器环境，支持直接ssh连接""" + try: + # 使用credential_id作为密钥文件名 + key_filename = f"id_rsa_{credential_id}" + key_path = os.path.join(self.ssh_dir, key_filename) + + # 写入私钥文件 + with open(key_path, 'w') as f: + f.write(private_key) + + # 设置私钥文件权限 + os.chmod(key_path, 0o600) + + # 如果有passphrase，发出提醒 + if passphrase: + logger.warning(f"SSH密钥 {credential_id} 有密码保护，在CI/CD自动化环境中可能需要在脚本中通过expect等工具处理") + + # 创建或更新SSH配置，支持全局使用 + self.update_ssh_config_global(credential_id, key_filename) + + logger.info(f"SSH密钥 {credential_id} 部署成功") + return True, "SSH密钥部署成功，现在可以直接使用 ssh user@host 进行连接" + + except Exception as e: + logger.error(f"部署SSH密钥失败: {str(e)}", exc_info=True) + return False, f"部署失败: {str(e)}" + + + def update_ssh_config_global(self, credential_id, key_filename): + """更新SSH配置文件，支持全局密钥使用""" + ssh_config_file = os.path.join(self.ssh_dir, 'config') + + # 构建配置条目 - 使用通配符Host *，使所有连接都能使用这些密钥 + config_entry = f""" +# SSH Key Credential: {credential_id} +Host * + IdentityFile ~/.ssh/{key_filename} + IdentitiesOnly no + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + LogLevel ERROR + # 禁用locale转发，避免远程服务器locale警告 + SendEnv -LC_* + # CI/CD性能优化 + Compression yes + ServerAliveInterval 60 + ServerAliveCountMax 3 + ConnectTimeout 10 + # 禁用不必要的认证方式 + GSSAPIAuthentication no + +""" + + # 读取现有配置 + existing_config = "" + if os.path.exists(ssh_config_file): + with open(ssh_config_file, 'r') as f: + existing_config = f.read() + + lines = existing_config.split('\n') + new_lines = [] + skip_section = False + + for line in lines: + if line.strip() == f"# SSH Key Credential: {credential_id}": + skip_section = True + continue + elif skip_section and line.strip() == "": + skip_section = False + continue + elif not skip_section: + new_lines.append(line) + + # 添加新的配置条目到开头 + new_config = config_entry + '\n'.join(new_lines).rstrip() + + # 写入配置文件 + with open(ssh_config_file, 'w') as f: + f.write(new_config) + + # 设置配置文件权限 + os.chmod(ssh_config_file, 0o600) + + def remove_ssh_key(self, credential_id): + """移除SSH密钥和配置""" + try: + # 删除私钥文件 + key_filename = f"id_rsa_{credential_id}" + key_path = os.path.join(self.ssh_dir, key_filename) + if os.path.exists(key_path): + os.remove(key_path) + + ssh_config_file = os.path.join(self.ssh_dir, 'config') + if os.path.exists(ssh_config_file): + with open(ssh_config_file, 'r') as f: + existing_config = f.read() + + lines = existing_config.split('\n') + new_lines = [] + skip_section = False + + for line in lines: + if line.strip() == f"# SSH Key Credential: {credential_id}": + skip_section = True + continue + elif skip_section and line.strip() == "": + skip_section = False + continue + elif not skip_section: + new_lines.append(line) + + new_config = '\n'.join(new_lines).rstrip() + with open(ssh_config_file, 'w') as f: + f.write(new_config) + + logger.info(f"SSH密钥 {credential_id} 清理成功") + return True, "SSH密钥清理成功" + + except Exception as e: + logger.error(f"清理SSH密钥失败: {str(e)}", exc_info=True) + return False, f"清理失败: {str(e)}" + + def get_deployment_status(self, credential_id): + """获取SSH密钥部署状态""" + key_filename = f"id_rsa_{credential_id}" + key_path = os.path.join(self.ssh_dir, key_filename) + + if os.path.exists(key_path): + # 检查配置文件中是否有对应条目 + ssh_config_file = os.path.join(self.ssh_dir, 'config') + if os.path.exists(ssh_config_file): + with open(ssh_config_file, 'r') as f: + config_content = f.read() + if f"# SSH Key Credential: {credential_id}" in config_content: + return True, "已部署" + return False, "配置缺失" + return False, "未部署" + + +class KubeconfigManager: + """Kubeconfig配置管理器 - 支持多集群配置合并""" + + def __init__(self): + self.kube_dir = os.path.expanduser('~/.kube') + self.config_file = os.path.join(self.kube_dir, 'config') + self.backup_dir = os.path.join(self.kube_dir, 'backups') + self.ensure_kube_dir() + + def ensure_kube_dir(self): + """确保kubectl目录存在并设置正确权限""" + if not os.path.exists(self.kube_dir): + os.makedirs(self.kube_dir, mode=0o700) + else: + os.chmod(self.kube_dir, 0o700) + + # 确保备份目录存在 + if not os.path.exists(self.backup_dir): + os.makedirs(self.backup_dir, mode=0o700) + + def parse_kubeconfig(self, content): + """解析kubeconfig内容，提取集群和上下文信息""" + try: + config = yaml.safe_load(content) + + # 验证基本结构 + if not isinstance(config, dict): + return None, "无效的kubeconfig格式：根节点必须是对象" + + required_keys = ['apiVersion', 'kind', 'clusters', 'users', 'contexts'] + missing_keys = [key for key in required_keys if key not in config] + if missing_keys: + return None, f"缺少必需的字段: {', '.join(missing_keys)}" + + # 提取信息 + clusters = config.get('clusters', []) + contexts = config.get('contexts', []) + current_context = config.get('current-context') + + if not clusters: + return None, "kubeconfig中没有找到集群配置" + + if not contexts: + return None, "kubeconfig中没有找到上下文配置" + + # 提取第一个集群和上下文的名称 + cluster_name = clusters[0].get('name', 'unknown-cluster') if clusters else 'unknown-cluster' + context_name = current_context or (contexts[0].get('name', 'unknown-context') if contexts else 'unknown-context') + + return { + 'cluster_name': cluster_name, + 'context_name': context_name, + 'clusters': [c.get('name') for c in clusters], + 'contexts': [c.get('name') for c in contexts], + 'current_context': current_context + }, None + + except yaml.YAMLError as e: + return None, f"YAML格式错误: {str(e)}" + except Exception as e: + return None, f"解析kubeconfig失败: {str(e)}" + + def backup_current_config(self): + """备份当前的kubeconfig文件""" + if os.path.exists(self.config_file): + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + backup_file = os.path.join(self.backup_dir, f'config_backup_{timestamp}') + shutil.copy2(self.config_file, backup_file) + logger.info(f"kubeconfig已备份到: {backup_file}") + return backup_file + return None + + def merge_kubeconfigs(self, credential_configs): + """合并多个kubeconfig配置""" + try: + merged_config = { + 'apiVersion': 'v1', + 'kind': 'Config', + 'clusters': [], + 'users': [], + 'contexts': [], + 'current-context': None + } + + all_clusters = {} + all_users = {} + all_contexts = {} + first_context = None + + # 读取现有配置 + if os.path.exists(self.config_file): + try: + with open(self.config_file, 'r') as f: + existing_config = yaml.safe_load(f) + + if existing_config and isinstance(existing_config, dict): + # 保留现有的集群、用户和上下文（非LiteOps管理的） + for cluster in existing_config.get('clusters', []): + cluster_name = cluster.get('name') + if cluster_name and not self._is_liteops_managed(cluster_name): + all_clusters[cluster_name] = cluster + + for user in existing_config.get('users', []): + user_name = user.get('name') + if user_name and not self._is_liteops_managed(user_name): + all_users[user_name] = user + + for context in existing_config.get('contexts', []): + context_name = context.get('name') + if context_name and not self._is_liteops_managed(context_name): + all_contexts[context_name] = context + + # 保留现有的current-context（不是LiteOps管理的） + current_context = existing_config.get('current-context') + if current_context and not self._is_liteops_managed(current_context): + merged_config['current-context'] = current_context + + except Exception as e: + logger.warning(f"读取现有kubeconfig失败，将创建新配置: {str(e)}") + + # 合并新的配置 + for config_content in credential_configs: + try: + config = yaml.safe_load(config_content) + + # 添加集群 + for cluster in config.get('clusters', []): + cluster_name = cluster.get('name') + if cluster_name: + # 添加LiteOps标记 + cluster_copy = cluster.copy() + cluster_copy['name'] = f"liteops-{cluster_name}" + all_clusters[cluster_copy['name']] = cluster_copy + + # 添加用户 + for user in config.get('users', []): + user_name = user.get('name') + if user_name: + # 添加LiteOps标记 + user_copy = user.copy() + user_copy['name'] = f"liteops-{user_name}" + all_users[user_copy['name']] = user_copy + + # 添加上下文 + for context in config.get('contexts', []): + context_name = context.get('name') + if context_name and context.get('context'): + # 更新引用并添加LiteOps标记 + context_copy = context.copy() + context_copy['name'] = f"liteops-{context_name}" + + # 更新context中的cluster和user引用 + context_obj = context_copy['context'].copy() + if 'cluster' in context_obj: + context_obj['cluster'] = f"liteops-{context_obj['cluster']}" + if 'user' in context_obj: + context_obj['user'] = f"liteops-{context_obj['user']}" + context_copy['context'] = context_obj + + all_contexts[context_copy['name']] = context_copy + + # 记录第一个上下文作为默认值 + if first_context is None: + first_context = context_copy['name'] + + except Exception as e: + logger.error(f"处理单个kubeconfig配置失败: {str(e)}") + continue + + # 组装最终配置 + merged_config['clusters'] = list(all_clusters.values()) + merged_config['users'] = list(all_users.values()) + merged_config['contexts'] = list(all_contexts.values()) + + # 设置默认上下文 + if merged_config['current-context'] is None and first_context: + merged_config['current-context'] = first_context + + return merged_config, None + + except Exception as e: + logger.error(f"合并kubeconfig失败: {str(e)}", exc_info=True) + return None, f"合并失败: {str(e)}" + + def _is_liteops_managed(self, name): + """检查资源是否由LiteOps管理""" + return name.startswith('liteops-') + + def deploy_single_kubeconfig(self, credential_id): + """部署单个kubeconfig凭证，自动与已有凭证合并""" + try: + # 获取指定的kubeconfig凭证 + try: + credential = KubeconfigCredential.objects.get(credential_id=credential_id) + except KubeconfigCredential.DoesNotExist: + return False, "指定的kubeconfig凭证不存在" + + # 获取所有已部署的kubeconfig凭证（通过检查当前config文件） + deployed_credentials = self.get_deployed_credentials() + + # 添加当前要部署的凭证 + if credential_id not in deployed_credentials: + deployed_credentials.append(credential_id) + + # 获取所有需要部署的凭证内容 + config_contents = [] + contexts = [] + + for cred_id in deployed_credentials: + try: + cred = KubeconfigCredential.objects.get(credential_id=cred_id) + config_contents.append(cred.kubeconfig_content) + contexts.append(f"liteops-{cred.context_name}") + except KubeconfigCredential.DoesNotExist: + continue + + if not config_contents: + return False, "没有有效的kubeconfig凭证可部署" + + # 备份当前配置 + backup_file = self.backup_current_config() + + # 合并配置 + merged_config, error = self.merge_kubeconfigs(config_contents) + if error: + return False, error + + # 写入合并后的配置 + with open(self.config_file, 'w') as f: + yaml.dump(merged_config, f, default_flow_style=False, allow_unicode=True) + + # 设置文件权限 + os.chmod(self.config_file, 0o600) + + logger.info(f"Kubeconfig凭证 {credential_id} 部署成功，当前包含 {len(contexts)} 个集群上下文") + return True, f"部署成功，当前配置包含 {len(contexts)} 个集群" + + except Exception as e: + logger.error(f"部署kubeconfig凭证失败: {str(e)}", exc_info=True) + return False, f"部署失败: {str(e)}" + + def get_deployed_credentials(self): + """获取当前已部署的凭证ID列表""" + deployed_creds = [] + + try: + if not os.path.exists(self.config_file): + return deployed_creds + + with open(self.config_file, 'r') as f: + config = yaml.safe_load(f) + + if not config or not isinstance(config, dict): + return deployed_creds + + # 通过上下文名称反推凭证ID + contexts = config.get('contexts', []) + for context in contexts: + context_name = context.get('name', '') + if context_name.startswith('liteops-'): + # 从上下文名称中提取原始名称，然后查找对应的凭证 + original_context = context_name[8:] # 移除 'liteops-' 前缀 + try: + cred = KubeconfigCredential.objects.get(context_name=original_context) + if cred.credential_id not in deployed_creds: + deployed_creds.append(cred.credential_id) + except KubeconfigCredential.DoesNotExist: + pass + + except Exception as e: + logger.error(f"获取已部署凭证列表失败: {str(e)}") + + return deployed_creds + + def undeploy_single_kubeconfig(self, credential_id): + """取消部署单个kubeconfig凭证""" + try: + # 获取当前已部署的凭证列表 + deployed_credentials = self.get_deployed_credentials() + + # 移除指定的凭证 + if credential_id in deployed_credentials: + deployed_credentials.remove(credential_id) + + # 备份现有配置 + backup_file = self.backup_current_config() + + if not deployed_credentials: + # 如果没有其他凭证了，清理所有LiteOps配置 + return self._cleanup_liteops_config() + else: + # 重新部署剩余的凭证 + config_contents = [] + for cred_id in deployed_credentials: + try: + cred = KubeconfigCredential.objects.get(credential_id=cred_id) + config_contents.append(cred.kubeconfig_content) + except KubeconfigCredential.DoesNotExist: + continue + + # 合并剩余配置 + merged_config, error = self.merge_kubeconfigs(config_contents) + if error: + return False, error + + # 写入更新后的配置 + with open(self.config_file, 'w') as f: + yaml.dump(merged_config, f, default_flow_style=False, allow_unicode=True) + + os.chmod(self.config_file, 0o600) + + logger.info(f"Kubeconfig凭证 {credential_id} 取消部署成功") + return True, "取消部署成功" + + except Exception as e: + logger.error(f"取消部署kubeconfig凭证失败: {str(e)}", exc_info=True) + return False, f"取消部署失败: {str(e)}" + + def _cleanup_liteops_config(self): + """清理所有LiteOps管理的配置""" + try: + config_file = self.config_file + if os.path.exists(config_file): + # 读取现有配置 + with open(config_file, 'r') as f: + config = yaml.safe_load(f) + + if config and isinstance(config, dict): + # 移除所有LiteOps管理的资源 + new_config = { + 'apiVersion': 'v1', + 'kind': 'Config', + 'clusters': [], + 'users': [], + 'contexts': [], + 'current-context': None + } + + # 保留非LiteOps管理的资源 + for cluster in config.get('clusters', []): + if not cluster.get('name', '').startswith('liteops-'): + new_config['clusters'].append(cluster) + + for user in config.get('users', []): + if not user.get('name', '').startswith('liteops-'): + new_config['users'].append(user) + + for context in config.get('contexts', []): + if not context.get('name', '').startswith('liteops-'): + new_config['contexts'].append(context) + + # 设置current-context + current_context = config.get('current-context') + if current_context and not current_context.startswith('liteops-'): + new_config['current-context'] = current_context + elif new_config['contexts']: + new_config['current-context'] = new_config['contexts'][0]['name'] + + # 写入更新后的配置 + with open(config_file, 'w') as f: + yaml.dump(new_config, f, default_flow_style=False, allow_unicode=True) + else: + # 如果配置无效，删除文件 + os.remove(config_file) + + return True, "清理成功" + + except Exception as e: + logger.error(f"清理LiteOps配置失败: {str(e)}") + return False, f"清理失败: {str(e)}" + + def deploy_kubeconfigs(self): + """部署所有kubeconfig凭证到~/.kube/config""" + try: + # 获取所有kubeconfig凭证 + credentials = KubeconfigCredential.objects.all() + + if not credentials: + return True, "没有需要部署的kubeconfig凭证" + + # 备份当前配置 + backup_file = self.backup_current_config() + + # 收集所有kubeconfig内容 + config_contents = [] + contexts = [] + + for credential in credentials: + config_contents.append(credential.kubeconfig_content) + contexts.append(f"liteops-{credential.context_name}") + + # 合并配置 + merged_config, error = self.merge_kubeconfigs(config_contents) + if error: + return False, error + + # 写入合并后的配置 + with open(self.config_file, 'w') as f: + yaml.dump(merged_config, f, default_flow_style=False, allow_unicode=True) + + # 设置文件权限 + os.chmod(self.config_file, 0o600) + + logger.info(f"Kubeconfig部署成功，包含 {len(contexts)} 个集群上下文") + return True, "部署成功" + + except Exception as e: + logger.error(f"部署kubeconfig失败: {str(e)}", exc_info=True) + return False, f"部署失败: {str(e)}" + + def get_deployment_status(self, credential_id=None): + """获取kubeconfig部署状态""" + try: + if not os.path.exists(self.config_file): + return False, "未部署" + + # 检查是否有LiteOps管理的上下文 + with open(self.config_file, 'r') as f: + config = yaml.safe_load(f) + + if not config or not isinstance(config, dict): + return False, "配置无效" + + if credential_id: + # 检查特定凭证的部署状态 + try: + credential = KubeconfigCredential.objects.get(credential_id=credential_id) + expected_context = f"liteops-{credential.context_name}" + + contexts = config.get('contexts', []) + for context in contexts: + if context.get('name') == expected_context: + return True, "已部署" + + return False, "未部署" + except KubeconfigCredential.DoesNotExist: + return False, "凭证不存在" + else: + # 检查全局部署状态 + contexts = config.get('contexts', []) + liteops_contexts = [ctx for ctx in contexts if ctx.get('name', '').startswith('liteops-')] + + if liteops_contexts: + return True, f"已部署 ({len(liteops_contexts)} 个集群)" + else: + return False, "未部署" + + except Exception as e: + logger.error(f"检查kubeconfig部署状态失败: {str(e)}") + return False, "状态未知" + +@method_decorator(csrf_exempt, name='dispatch') +class CredentialView(View): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ssh_manager = SSHKeyManager() + self.kubeconfig_manager = KubeconfigManager() + + @method_decorator(jwt_auth_required) + def get(self, request): + """获取凭证列表""" + try: + credential_type = request.GET.get('type') + if credential_type not in CREDENTIAL_MODELS: + return JsonResponse({ + 'code': 400, + 'message': '无效的凭证类型' + }) + + model = CREDENTIAL_MODELS[credential_type] + credentials = model.objects.all() + + data = [] + for credential in credentials: + item = { + 'credential_id': credential.credential_id, + 'name': credential.name, + 'description': credential.description, + 'creator': { + 'user_id': credential.creator.user_id, + 'name': credential.creator.name + }, + 'create_time': credential.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'update_time': credential.update_time.strftime('%Y-%m-%d %H:%M:%S'), + } + + # 根据不同凭证类型添加特定字段 + if credential_type == 'gitlab_token': + pass # GitLab Token没有额外字段 + elif credential_type == 'ssh_key': + # 添加部署状态 + deployed, status = self.ssh_manager.get_deployment_status(credential.credential_id) + item['deployed'] = deployed + item['deploy_status'] = status + elif credential_type == 'kubeconfig': + # 添加集群和上下文信息 + item['cluster_name'] = credential.cluster_name + item['context_name'] = credential.context_name + # 添加单个凭证的部署状态 + deployed, status = self.kubeconfig_manager.get_deployment_status(credential.credential_id) + item['deployed'] = deployed + item['deploy_status'] = status + + data.append(item) + + return JsonResponse({ + 'code': 200, + 'message': '获取凭证列表成功', + 'data': data + }) + except Exception as e: + logger.error(f'获取凭证列表失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def post(self, request): + """创建凭证或执行部署操作""" + try: + data = json.loads(request.body) + action = data.get('action', 'create') + + if action == 'deploy': + return self.deploy_ssh_key(request, data) + elif action == 'undeploy': + return self.undeploy_ssh_key(request, data) + elif action == 'deploy_kubeconfig': + return self.deploy_kubeconfig(request, data) + elif action == 'undeploy_kubeconfig': + return self.undeploy_kubeconfig(request, data) + else: + return self.create_credential(request, data) + + except Exception as e: + logger.error(f'处理请求失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + def create_credential(self, request, data): + """创建凭证""" + credential_type = data.get('type') + + if credential_type not in CREDENTIAL_MODELS: + return JsonResponse({ + 'code': 400, + 'message': '无效的凭证类型' + }) + + # 获取当前用户 + try: + creator = User.objects.get(user_id=request.user_id) + except User.DoesNotExist: + return JsonResponse({ + 'code': 400, + 'message': '用户不存在' + }) + + model = CREDENTIAL_MODELS[credential_type] + credential = model( + credential_id=generate_id(), + name=data.get('name'), + description=data.get('description'), + creator=creator + ) + + # 根据不同凭证类型设置特定字段 + if credential_type == 'gitlab_token': + credential.token = data.get('token') # GitLab Token 不加密 + elif credential_type == 'ssh_key': + credential.private_key = data.get('private_key') + credential.passphrase = data.get('passphrase') + elif credential_type == 'kubeconfig': + kubeconfig_content = data.get('kubeconfig_content') + if kubeconfig_content: + # 解析kubeconfig内容获取集群和上下文信息 + parse_result, error = self.kubeconfig_manager.parse_kubeconfig(kubeconfig_content) + if error: + return JsonResponse({ + 'code': 400, + 'message': f'Kubeconfig格式错误: {error}' + }) + + credential.kubeconfig_content = kubeconfig_content + credential.cluster_name = parse_result['cluster_name'] + credential.context_name = parse_result['context_name'] + + credential.save() + + return JsonResponse({ + 'code': 200, + 'message': '创建凭证成功', + 'data': { + 'credential_id': credential.credential_id + } + }) + + def deploy_ssh_key(self, request, data): + """部署SSH密钥""" + credential_id = data.get('credential_id') + + if not credential_id: + return JsonResponse({ + 'code': 400, + 'message': '凭证ID不能为空' + }) + + try: + credential = SSHKeyCredential.objects.get(credential_id=credential_id) + except SSHKeyCredential.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': 'SSH密钥凭证不存在' + }) + + # 执行部署 + success, message = self.ssh_manager.deploy_ssh_key( + credential_id=credential.credential_id, + private_key=credential.private_key, + passphrase=credential.passphrase + ) + + if success: + return JsonResponse({ + 'code': 200, + 'message': message, + 'data': { + 'credential_id': credential_id, + 'usage_example': 'ssh root@your-server-ip', + 'key_file': f'~/.ssh/id_rsa_{credential_id}' + } + }) + else: + return JsonResponse({ + 'code': 500, + 'message': message + }) + + def undeploy_ssh_key(self, request, data): + """取消部署SSH密钥""" + credential_id = data.get('credential_id') + + if not credential_id: + return JsonResponse({ + 'code': 400, + 'message': '凭证ID不能为空' + }) + + success, message = self.ssh_manager.remove_ssh_key(credential_id) + + if success: + return JsonResponse({ + 'code': 200, + 'message': message + }) + else: + return JsonResponse({ + 'code': 500, + 'message': message + }) + + def deploy_kubeconfig(self, request, data): + """部署单个kubeconfig凭证""" + credential_id = data.get('credential_id') + + if not credential_id: + return JsonResponse({ + 'code': 400, + 'message': '凭证ID不能为空' + }) + + # 执行部署 + success, message = self.kubeconfig_manager.deploy_single_kubeconfig(credential_id) + + if success: + return JsonResponse({ + 'code': 200, + 'message': message + }) + else: + return JsonResponse({ + 'code': 500, + 'message': message + }) + + def undeploy_kubeconfig(self, request, data): + """取消部署单个kubeconfig凭证""" + credential_id = data.get('credential_id') + + if not credential_id: + return JsonResponse({ + 'code': 400, + 'message': '凭证ID不能为空' + }) + + success, message = self.kubeconfig_manager.undeploy_single_kubeconfig(credential_id) + + if success: + return JsonResponse({ + 'code': 200, + 'message': message + }) + else: + return JsonResponse({ + 'code': 500, + 'message': message + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """更新凭证""" + try: + data = json.loads(request.body) + credential_type = data.get('type') + credential_id = data.get('credential_id') + + if not credential_id: + return JsonResponse({ + 'code': 400, + 'message': '凭证ID不能为空' + }) + + if credential_type not in CREDENTIAL_MODELS: + return JsonResponse({ + 'code': 400, + 'message': '无效的凭证类型' + }) + + model = CREDENTIAL_MODELS[credential_type] + try: + credential = model.objects.get(credential_id=credential_id) + except model.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '凭证不存在' + }) + + # 更新基本字段 + credential.name = data.get('name', credential.name) + credential.description = data.get('description', credential.description) + + # 根据不同凭证类型更新特定字段 + if credential_type == 'gitlab_token': + if 'token' in data: # 只在提供新token时更新 + credential.token = data['token'] # GitLab Token 不加密 + elif credential_type == 'ssh_key': + if 'private_key' in data: # 只在提供新私钥时更新 + credential.private_key = data['private_key'] + # 如果已部署，需要重新部署 + deployed, _ = self.ssh_manager.get_deployment_status(credential_id) + if deployed: + self.ssh_manager.deploy_ssh_key( + credential_id=credential.credential_id, + private_key=credential.private_key, + passphrase=credential.passphrase + ) + if 'passphrase' in data: # 只在提供新密码时更新 + credential.passphrase = data['passphrase'] + elif credential_type == 'kubeconfig': + if 'kubeconfig_content' in data: # 只在提供新kubeconfig内容时更新 + kubeconfig_content = data['kubeconfig_content'] + # 解析kubeconfig内容获取集群和上下文信息 + parse_result, error = self.kubeconfig_manager.parse_kubeconfig(kubeconfig_content) + if error: + return JsonResponse({ + 'code': 400, + 'message': f'Kubeconfig格式错误: {error}' + }) + + credential.kubeconfig_content = kubeconfig_content + credential.cluster_name = parse_result['cluster_name'] + credential.context_name = parse_result['context_name'] + + # 如果已部署，需要重新部署所有kubeconfig凭证 + deployed, _ = self.kubeconfig_manager.get_deployment_status() + if deployed: + self.kubeconfig_manager.deploy_kubeconfigs() + + credential.save() + + return JsonResponse({ + 'code': 200, + 'message': '更新凭证成功' + }) + except Exception as e: + logger.error(f'更新凭证失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def delete(self, request): + """删除凭证""" + try: + data = json.loads(request.body) + credential_type = data.get('type') + credential_id = data.get('credential_id') + + if not credential_id: + return JsonResponse({ + 'code': 400, + 'message': '凭证ID不能为空' + }) + + if credential_type not in CREDENTIAL_MODELS: + return JsonResponse({ + 'code': 400, + 'message': '无效的凭证类型' + }) + + model = CREDENTIAL_MODELS[credential_type] + try: + credential = model.objects.get(credential_id=credential_id) + + # 如果是SSH密钥，先清理部署的密钥 + if credential_type == 'ssh_key': + self.ssh_manager.remove_ssh_key(credential_id) + elif credential_type == 'kubeconfig': + # 删除kubeconfig凭证后，重新部署剩余的配置 + credential.delete() + self.kubeconfig_manager.undeploy_single_kubeconfig(credential_id) + return JsonResponse({ + 'code': 200, + 'message': '删除凭证成功，kubeconfig配置已更新' + }) + + credential.delete() + except model.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '凭证不存在' + }) + + return JsonResponse({ + 'code': 200, + 'message': '删除凭证成功' + }) + except Exception as e: + logger.error(f'删除凭证失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/dashboard.py b/backend/apps/views/dashboard.py new file mode 100644 index 0000000..7c691c7 --- /dev/null +++ b/backend/apps/views/dashboard.py @@ -0,0 +1,288 @@ +import logging +from datetime import datetime, timedelta +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.db.models import Count +from ..models import Project, BuildTask, BuildHistory, User, Environment + +logger = logging.getLogger('apps') + +@method_decorator(csrf_exempt, name='dispatch') +class DashboardStatsView(View): + """首页统计数据接口""" + + def get(self, request): + """获取首页统计数据""" + try: + # 获取项目总数 + project_count = Project.objects.count() + + # 获取构建任务总数 + task_count = BuildTask.objects.count() + + # 获取用户总数 + user_count = User.objects.count() + + # 获取环境总数 + env_count = Environment.objects.count() + + # 获取总构建数量 + total_builds_count = BuildHistory.objects.count() + + # 获取最近7天的构建成功率 + seven_days_ago = datetime.now() - timedelta(days=7) + recent_builds = BuildHistory.objects.filter(create_time__gte=seven_days_ago) + total_recent_builds = recent_builds.count() + success_recent_builds = recent_builds.filter(status='success').count() + + success_rate = 0 + if total_recent_builds > 0: + success_rate = round((success_recent_builds / total_recent_builds) * 100, 2) + + return JsonResponse({ + 'code': 200, + 'message': '获取首页统计数据成功', + 'data': { + 'project_count': project_count, + 'task_count': task_count, + 'user_count': user_count, + 'env_count': env_count, + 'total_builds_count': total_builds_count, + 'success_rate': success_rate, + 'total_recent_builds': total_recent_builds + } + }) + except Exception as e: + logger.error(f'获取首页统计数据失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + +@method_decorator(csrf_exempt, name='dispatch') +class BuildTrendView(View): + """构建任务趋势接口""" + + def get(self, request): + """获取构建任务趋势数据""" + try: + # 获取时间范围参数，默认为最近7天 + days = int(request.GET.get('days', 7)) + + # 计算日期范围：包含今天在内的最近 days 天 + today = datetime.now().date() # 获取今天的日期部分 + start_date = today - timedelta(days=days - 1) # 开始日期是今天往前 days-1 天 + + # 准备日期列表和结果数据 + date_list = [] + success_data = [] + failed_data = [] + + # 生成从 start_date 到 today 的日期列表 + current_date = start_date + while current_date <= today: + date_str = current_date.strftime('%Y-%m-%d') + date_list.append(date_str) + + # 查询当天的构建数据 + day_start = datetime.combine(current_date, datetime.min.time()) + day_end = datetime.combine(current_date, datetime.max.time()) + + # 成功构建数 + success_count = BuildHistory.objects.filter( + create_time__gte=day_start, + create_time__lte=day_end, + status='success' + ).count() + + # 失败构建数 + failed_count = BuildHistory.objects.filter( + create_time__gte=day_start, + create_time__lte=day_end, + status='failed' + ).count() + + success_data.append(success_count) + failed_data.append(failed_count) + + current_date += timedelta(days=1) + + return JsonResponse({ + 'code': 200, + 'message': '获取构建任务趋势数据成功', + 'data': { + 'dates': date_list, + 'success': success_data, + 'failed': failed_data + } + }) + except Exception as e: + logger.error(f'获取构建任务趋势数据失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + +@method_decorator(csrf_exempt, name='dispatch') +class BuildDetailView(View): + """构建详细数据接口""" + + def get(self, request): + """获取指定日期的构建详细数据""" + try: + # 获取日期参数 + date_str = request.GET.get('date') + if not date_str: + return JsonResponse({ + 'code': 400, + 'message': '日期参数不能为空' + }) + + # 解析日期 + try: + date = datetime.strptime(date_str, '%Y-%m-%d') + day_start = datetime(date.year, date.month, date.day, 0, 0, 0) + day_end = datetime(date.year, date.month, date.day, 23, 59, 59) + except ValueError: + return JsonResponse({ + 'code': 400, + 'message': '日期格式不正确，应为YYYY-MM-DD' + }) + + # 查询当天的构建历史 + builds = BuildHistory.objects.filter( + create_time__gte=day_start, + create_time__lte=day_end + ).select_related('task', 'operator').order_by('-create_time') + + build_list = [] + for build in builds: + # 计算构建耗时 + duration = '未完成' + if build.build_time and 'total_duration' in build.build_time: + total_seconds = int(build.build_time.get('total_duration', 0)) + minutes = total_seconds // 60 + seconds = total_seconds % 60 + duration = f"{minutes}分{seconds}秒" + + build_list.append({ + 'id': build.history_id, + 'build_number': build.build_number, + 'task_name': build.task.name, + 'status': build.status, + 'branch': build.branch, + 'version': build.version, + 'start_time': build.build_time.get('start_time') if build.build_time else build.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'duration': duration, + 'operator': build.operator.name if build.operator else '系统' + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取构建详细数据成功', + 'data': build_list + }) + except Exception as e: + logger.error(f'获取构建详细数据失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + +@method_decorator(csrf_exempt, name='dispatch') +class RecentBuildsView(View): + """最近构建任务接口""" + + def get(self, request): + """获取最近构建任务数据""" + try: + # 获取数量参数，默认为10条 + limit = int(request.GET.get('limit', 5)) + + # 查询最近的构建历史 + recent_builds = BuildHistory.objects.select_related( + 'task', 'task__environment', 'operator' # 关联环境信息 + ).order_by('-create_time')[:limit] + + build_list = [] + for build in recent_builds: + # 计算构建耗时 + duration = '未完成' + if build.build_time and 'total_duration' in build.build_time: + total_seconds = int(build.build_time.get('total_duration', 0)) + minutes = total_seconds // 60 + seconds = total_seconds % 60 + duration = f"{minutes}分 {seconds}秒" + + build_list.append({ + 'id': build.history_id, + 'build_number': build.build_number, + 'task_name': build.task.name, + 'status': build.status, + 'branch': build.branch, + 'version': build.version, + 'environment': build.task.environment.name if build.task.environment else None, # 添加环境名称 + 'requirement': build.requirement, + 'start_time': build.build_time.get('start_time') if build.build_time else build.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'duration': duration, + 'operator': build.operator.name if build.operator else '系统' + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取最近构建任务数据成功', + 'data': build_list + }) + except Exception as e: + logger.error(f'获取最近构建任务数据失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + +@method_decorator(csrf_exempt, name='dispatch') +class ProjectDistributionView(View): + """项目分布接口""" + + def get(self, request): + """获取项目分布数据""" + try: + # 按项目类别统计 + category_stats = Project.objects.values('category').annotate(count=Count('id')) + + # 格式化数据 + category_data = [] + for stat in category_stats: + category = stat['category'] or '未分类' + category_data.append({ + 'type': self._get_category_name(category), + 'value': stat['count'] + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取项目分布数据成功', + 'data': category_data + }) + except Exception as e: + logger.error(f'获取项目分布数据失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + def _get_category_name(self, category): + """获取项目类别名称""" + category_map = { + 'frontend': '前端项目', + 'backend': '后端项目', + 'mobile': '移动端项目', + 'other': '其他项目' + } + return category_map.get(category, '未分类') \ No newline at end of file diff --git a/backend/apps/views/environment.py b/backend/apps/views/environment.py new file mode 100644 index 0000000..66f2a4c --- /dev/null +++ b/backend/apps/views/environment.py @@ -0,0 +1,337 @@ +import json +import uuid +import hashlib +import logging +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.db import transaction +from django.db.models import Q +from ..models import Environment, User +from ..utils.auth import jwt_auth_required +from ..utils.permissions import get_user_permissions + +logger = logging.getLogger('apps') + +def generate_id(): + """生成唯一ID""" + return hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:32] + +@method_decorator(csrf_exempt, name='dispatch') +class EnvironmentTypeView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取环境类型列表""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + + # 检查用户是否有环境查看权限 + function_permissions = user_permissions.get('function', {}) + environment_permissions = function_permissions.get('environment', []) + + if 'view' not in environment_permissions: + logger.warning(f'用户[{request.user_id}]没有环境查看权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看环境' + }, status=403) + + # 获取所有环境 + query = Q() + + # 应用环境权限过滤 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_scope == 'custom': + permitted_environment_types = data_permissions.get('environment_types', []) + if not permitted_environment_types: + # 如果设置了自定义环境权限但列表为空，意味着没有权限查看任何环境 + logger.info(f'用户[{request.user_id}]没有权限查看任何环境') + return JsonResponse({ + 'code': 200, + 'message': '获取环境列表成功', + 'data': [] + }) + + # 限制只能查看有权限的环境类型 + query &= Q(type__in=permitted_environment_types) + + environments = Environment.objects.filter(query).order_by('name') + + # 格式化结果 + env_list = [] + for env in environments: + if env.type: + env_list.append({ + 'environment_id': env.environment_id, + 'type': env.type, + 'name': env.name + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取环境列表成功', + 'data': env_list + }) + except Exception as e: + logger.error(f'获取环境列表失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + +@method_decorator(csrf_exempt, name='dispatch') +class EnvironmentView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取环境列表""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + + # 检查用户是否有环境查看权限 + function_permissions = user_permissions.get('function', {}) + environment_permissions = function_permissions.get('environment', []) + + if 'view' not in environment_permissions: + logger.warning(f'用户[{request.user_id}]没有环境查看权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看环境' + }, status=403) + + environment_id = request.GET.get('environment_id') + name = request.GET.get('name') + type = request.GET.get('type') + page = int(request.GET.get('page', 1)) + page_size = int(request.GET.get('page_size', 10)) + + # 构建查询条件 + query = Q() + if environment_id: + query &= Q(environment_id=environment_id) + if name: + query &= Q(name__icontains=name) + if type: + query &= Q(type=type) + + # 应用环境权限过滤 + environment_scope = data_permissions.get('environment_scope', 'all') + if environment_scope == 'custom': + permitted_environment_types = data_permissions.get('environment_types', []) + if not permitted_environment_types: + logger.info(f'用户[{request.user_id}]没有权限查看任何环境') + return JsonResponse({ + 'code': 200, + 'message': '获取环境列表成功', + 'data': [], + 'total': 0, + 'page': page, + 'page_size': page_size + }) + + # 如果指定了环境类型，检查是否有权限 + if type and type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的环境类型[{type}]') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该类型的环境' + }, status=403) + + if environment_id: + try: + env = Environment.objects.get(environment_id=environment_id) + if env.type not in permitted_environment_types: + logger.warning(f'用户[{request.user_id}]尝试查看无权限的环境[{environment_id}]') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看该环境' + }, status=403) + except Environment.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '环境不存在' + }, status=404) + + # 限制只能查看有权限的环境类型 + query &= Q(type__in=permitted_environment_types) + + # 获取环境列表 + environments = Environment.objects.filter(query).select_related('creator') + + # 计算总数 + total = environments.count() + + # 分页 + start = (page - 1) * page_size + end = start + page_size + environments = environments[start:end] + + environment_list = [] + + for env in environments: + environment_list.append({ + 'environment_id': env.environment_id, + 'name': env.name, + 'type': env.type, + 'description': env.description, + 'creator': { + 'user_id': env.creator.user_id, + 'username': env.creator.username, + 'name': env.creator.name + } if env.creator else None, + 'create_time': env.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'update_time': env.update_time.strftime('%Y-%m-%d %H:%M:%S') + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取环境列表成功', + 'data': environment_list, + 'total': total, + 'page': page, + 'page_size': page_size + }) + except Exception as e: + logger.error(f'获取环境列表失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def post(self, request): + """创建环境""" + try: + with transaction.atomic(): + data = json.loads(request.body) + name = data.get('name') + type = data.get('type') + description = data.get('description') + + if not all([name, type]): + return JsonResponse({ + 'code': 400, + 'message': '环境名称和类型不能为空' + }) + + # 检查环境名称是否已存在 + if Environment.objects.filter(name=name).exists(): + return JsonResponse({ + 'code': 400, + 'message': '环境名称已存在' + }) + + # 创建环境 + creator = User.objects.get(user_id=request.user_id) + environment = Environment.objects.create( + environment_id=generate_id(), + name=name, + type=type, + description=description, + creator=creator + ) + + return JsonResponse({ + 'code': 200, + 'message': '创建环境成功', + 'data': { + 'environment_id': environment.environment_id, + 'name': environment.name + } + }) + except Exception as e: + logger.error(f'创建环境失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """编辑环境""" + try: + with transaction.atomic(): + data = json.loads(request.body) + environment_id = data.get('environment_id') + name = data.get('name') + type = data.get('type') + description = data.get('description') + + if not environment_id: + return JsonResponse({ + 'code': 400, + 'message': '环境ID不能为空' + }) + + try: + environment = Environment.objects.get(environment_id=environment_id) + except Environment.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '环境不存在' + }) + + # 检查名称是否已存在（排除当前环境） + if name and name != environment.name: + if Environment.objects.filter(name=name).exists(): + return JsonResponse({ + 'code': 400, + 'message': '环境名称已存在' + }) + environment.name = name + + if type: + environment.type = type + if description is not None: + environment.description = description + + environment.save() + + return JsonResponse({ + 'code': 200, + 'message': '更新环境成功' + }) + except Exception as e: + logger.error(f'更新环境失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def delete(self, request): + """删除环境""" + try: + with transaction.atomic(): + data = json.loads(request.body) + environment_id = data.get('environment_id') + + if not environment_id: + return JsonResponse({ + 'code': 400, + 'message': '环境ID不能为空' + }) + + try: + environment = Environment.objects.get(environment_id=environment_id) + environment.delete() + return JsonResponse({ + 'code': 200, + 'message': '删除环境成功' + }) + except Environment.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '环境不存在' + }) + + except Exception as e: + logger.error(f'删除环境失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/gitlab.py b/backend/apps/views/gitlab.py new file mode 100644 index 0000000..d880541 --- /dev/null +++ b/backend/apps/views/gitlab.py @@ -0,0 +1,183 @@ +import json +import logging +import gitlab +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from ..models import Project, BuildTask, GitlabTokenCredential +from ..utils.auth import jwt_auth_required + +logger = logging.getLogger('apps') + +def get_gitlab_client(repository, git_token=None): + """获取GitLab客户端""" + try: + if not git_token: + # 获取第一个可用的GitLab Token凭证 + credential = GitlabTokenCredential.objects.first() + if not credential: + raise ValueError('未找到GitLab Token凭证') + git_token = credential.token + + # 从仓库地址中提取GitLab实例URL + repository_parts = repository.split('/') + gitlab_url = '/'.join(repository_parts[:3]) # 获取到域名部分 + if not gitlab_url.startswith('http'): + gitlab_url = f'http://{gitlab_url}' + + # 创建GitLab客户端 + gl = gitlab.Gitlab( + url=gitlab_url, + private_token=git_token + ) + gl.auth() + return gl + except Exception as e: + logger.error(f'获取GitLab客户端失败: {str(e)}', exc_info=True) + raise + +def get_gitlab_project(repository, git_token=None): + """获取GitLab项目""" + try: + gl = get_gitlab_client(repository, git_token) + repository_parts = repository.split('/') + project_path = '/'.join(repository_parts[3:]) # 获取group/project部分 + project_path = project_path.replace('.git', '') + return gl.projects.get(project_path) + except Exception as e: + logger.error(f'获取GitLab项目失败: {str(e)}', exc_info=True) + raise + +@method_decorator(csrf_exempt, name='dispatch') +class GitlabBranchView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取Git分支列表""" + try: + task_id = request.GET.get('task_id') + if not task_id: + return JsonResponse({ + 'code': 400, + 'message': '缺少任务ID' + }) + + # 获取任务信息 + try: + task = BuildTask.objects.select_related('project', 'git_token').get(task_id=task_id) + except BuildTask.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '任务不存在' + }) + + if not task.project or not task.project.repository: + return JsonResponse({ + 'code': 400, + 'message': '任务未配置Git仓库' + }) + + # 获取GitLab项目 + gitlab_project = get_gitlab_project( + task.project.repository, + task.git_token.token if task.git_token else None + ) + + # 获取分支列表 + branches = gitlab_project.branches.list(all=True) + branch_list = [] + for branch in branches: + branch_list.append({ + 'name': branch.name, + 'protected': branch.protected, + 'merged': branch.merged, + 'default': branch.default, + 'commit': { + 'id': branch.commit['id'], + 'title': branch.commit['title'], + 'author_name': branch.commit['author_name'], + 'authored_date': branch.commit['authored_date'], + } + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取分支列表成功', + 'data': branch_list + }) + except Exception as e: + logger.error(f'获取分支列表失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + +@method_decorator(csrf_exempt, name='dispatch') +class GitlabCommitView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取Git提交记录""" + try: + task_id = request.GET.get('task_id') + branch = request.GET.get('branch') + + if not all([task_id, branch]): + return JsonResponse({ + 'code': 400, + 'message': '缺少必要参数' + }) + + # 获取任务信息 + try: + task = BuildTask.objects.select_related('project', 'git_token').get(task_id=task_id) + except BuildTask.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '任务不存在' + }) + + if not task.project or not task.project.repository: + return JsonResponse({ + 'code': 400, + 'message': '任务未配置Git仓库' + }) + + # 获取GitLab项目 + gitlab_project = get_gitlab_project( + task.project.repository, + task.git_token.token if task.git_token else None + ) + + # 获取最近的提交记录 + commits = gitlab_project.commits.list( + ref_name=branch, + all=False, + per_page=20, # 增加返回数量 + order_by='created_at' + ) + + commit_list = [] + for commit in commits: + commit_list.append({ + 'id': commit.id, + 'short_id': commit.short_id, + 'title': commit.title, + 'message': commit.message, + 'author_name': commit.author_name, + 'author_email': commit.author_email, + 'authored_date': commit.authored_date, + 'created_at': commit.created_at, + 'web_url': commit.web_url + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取提交记录成功', + 'data': commit_list + }) + except Exception as e: + logger.error(f'获取提交记录失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/login.py b/backend/apps/views/login.py new file mode 100644 index 0000000..b5f7a07 --- /dev/null +++ b/backend/apps/views/login.py @@ -0,0 +1,221 @@ +import json +import hashlib +import jwt +import uuid +from datetime import datetime, timedelta +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods +from django.conf import settings +from ..models import User, UserToken, LoginLog +from ..utils.security import SecurityValidator + +def generate_token_id(): + """生成token_id""" + return hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:32] + +def generate_log_id(): + """生成log_id""" + return hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:32] + +@csrf_exempt +@require_http_methods(["POST"]) +def login(request): + try: + data = json.loads(request.body) + username = data.get('username') + password = data.get('password') + + # 获取客户端IP和用户代理 + ip_address = request.META.get('HTTP_X_FORWARDED_FOR', request.META.get('REMOTE_ADDR', '')) + user_agent = request.META.get('HTTP_USER_AGENT', '') + + if not username or not password: + return JsonResponse({ + 'code': 400, + 'message': '用户名和密码不能为空' + }) + + # 密码加密 + password_hash = hashlib.sha256(password.encode()).hexdigest() + + try: + user = User.objects.get(username=username) + + # 记录登录日志 + log_id = generate_log_id() + + # 检查账户是否被锁定 + is_not_locked, lockout_message = SecurityValidator.check_account_lockout(user, ip_address) + if not is_not_locked: + # 账户被锁定，记录失败登录 + LoginLog.objects.create( + log_id=log_id, + user=user, + ip_address=ip_address, + user_agent=user_agent, + status='failed', + fail_reason=lockout_message + ) + + return JsonResponse({ + 'code': 423, # 423 Locked + 'message': lockout_message + }) + + if user.password != password_hash: + # 密码错误，记录失败尝试 + failed_attempts, max_attempts = SecurityValidator.record_failed_login(user, ip_address) + + # 记录失败登录日志 + LoginLog.objects.create( + log_id=log_id, + user=user, + ip_address=ip_address, + user_agent=user_agent, + status='failed', + fail_reason='密码错误' + ) + + # 提供剩余尝试次数信息 + remaining_attempts = max_attempts - failed_attempts + if remaining_attempts > 0: + message = f'密码错误，还可尝试{remaining_attempts}次' + else: + config = SecurityValidator.get_security_config() + message = f'密码错误次数过多，账户已被锁定{config.lockout_duration}分钟' + + return JsonResponse({ + 'code': 401, + 'message': message + }) + + if user.status == 0: + # 账号锁定，记录失败登录 + LoginLog.objects.create( + log_id=log_id, + user=user, + ip_address=ip_address, + user_agent=user_agent, + status='failed', + fail_reason='账号已被锁定' + ) + + return JsonResponse({ + 'code': 423, # 423 Locked + 'message': '账号已被锁定，请联系管理员解锁' + }) + + # 登录成功，重置失败尝试次数 + SecurityValidator.record_successful_login(user, ip_address) + + # 检查会话超时设置 + config = SecurityValidator.get_security_config() + + # 生成token_id和JWT token + token_id = generate_token_id() + token_payload = { + 'user_id': user.user_id, + 'username': user.username, + 'exp': datetime.utcnow() + timedelta(minutes=config.session_timeout) # 使用配置的会话超时时间 + } + token = jwt.encode(token_payload, settings.SECRET_KEY, algorithm='HS256') + + # 更新或创建token记录 + UserToken.objects.update_or_create( + user=user, + defaults={ + 'token_id': token_id, + 'token': token + } + ) + + # 更新登录时间 + user.login_time = datetime.now() + user.save() + + # 记录成功登录日志 + LoginLog.objects.create( + log_id=log_id, + user=user, + ip_address=ip_address, + user_agent=user_agent, + status='success' + ) + + return JsonResponse({ + 'code': 200, + 'message': '登录成功', + 'data': { + 'token_id': token_id, + 'token': token, + 'user': { + 'user_id': user.user_id, + 'username': user.username, + 'name': user.name, + 'email': user.email + } + } + }) + + except User.DoesNotExist: + # 用户不存在，记录失败登录 + log_id = generate_log_id() + LoginLog.objects.create( + log_id=log_id, + ip_address=ip_address, + user_agent=user_agent, + status='failed', + fail_reason='用户不存在' + ) + + return JsonResponse({ + 'code': 404, + 'message': '用户不存在' + }) + + except Exception as e: + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + +@csrf_exempt +@require_http_methods(["POST"]) +def logout(request): + try: + token = request.headers.get('Authorization') + if not token: + return JsonResponse({ + 'code': 400, + 'message': '未提供Token' + }) + + try: + # 解析token获取user_id + payload = jwt.decode(token, settings.SECRET_KEY, algorithms=['HS256']) + user_id = payload.get('user_id') + + # 删除该用户的token记录 + UserToken.objects.filter(user_id=user_id).delete() + + return JsonResponse({ + 'code': 200, + 'message': '退出成功' + }) + except jwt.ExpiredSignatureError: + return JsonResponse({ + 'code': 401, + 'message': 'Token已过期' + }) + except jwt.InvalidTokenError: + return JsonResponse({ + 'code': 401, + 'message': '无效的Token' + }) + + except Exception as e: + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/logs.py b/backend/apps/views/logs.py new file mode 100644 index 0000000..6738bd6 --- /dev/null +++ b/backend/apps/views/logs.py @@ -0,0 +1,121 @@ +import json +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods +from django.db.models import Q +from django.core.paginator import Paginator +from ..models import LoginLog, User +from ..utils.auth import jwt_auth_required + +@csrf_exempt +@jwt_auth_required +@require_http_methods(["GET"]) +def login_logs_list(request): + """ + 获取登录日志列表 + """ + try: + # 获取查询参数 + page = int(request.GET.get('page', 1)) + page_size = int(request.GET.get('page_size', 10)) + username = request.GET.get('username', '') + status = request.GET.get('status', '') + ip_address = request.GET.get('ip_address', '') + start_time = request.GET.get('start_time', '') + end_time = request.GET.get('end_time', '') + + # 构建查询条件 + query = Q() + + if username: + users = User.objects.filter(username__icontains=username) + query &= Q(user__in=users) + + if status: + query &= Q(status=status) + + if ip_address: + query &= Q(ip_address__icontains=ip_address) + + if start_time: + query &= Q(login_time__gte=start_time) + + if end_time: + query &= Q(login_time__lte=end_time) + + # 获取登录日志 + logs = LoginLog.objects.filter(query).select_related('user').order_by('-login_time') + + # 分页 + paginator = Paginator(logs, page_size) + current_page = paginator.page(page) + + # 格式化返回数据 + log_list = [] + for log in current_page.object_list: + log_data = { + 'log_id': log.log_id, + 'username': log.user.username if log.user else None, + 'user_id': log.user.user_id if log.user else None, + 'ip_address': log.ip_address, + 'user_agent': log.user_agent, + 'status': log.status, + 'fail_reason': log.fail_reason, + 'login_time': log.login_time.strftime('%Y-%m-%d %H:%M:%S') if log.login_time else None + } + log_list.append(log_data) + + return JsonResponse({ + 'code': 200, + 'message': '获取登录日志成功', + 'data': { + 'total': paginator.count, + 'page': page, + 'page_size': page_size, + 'logs': log_list + } + }) + except Exception as e: + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + +@csrf_exempt +@jwt_auth_required +@require_http_methods(["GET"]) +def login_log_detail(request, log_id): + """ + 获取登录日志详情 + """ + try: + try: + log = LoginLog.objects.select_related('user').get(log_id=log_id) + except LoginLog.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '登录日志不存在' + }) + + log_data = { + 'log_id': log.log_id, + 'username': log.user.username if log.user else None, + 'user_id': log.user.user_id if log.user else None, + 'user_name': log.user.name if log.user else None, + 'ip_address': log.ip_address, + 'user_agent': log.user_agent, + 'status': log.status, + 'fail_reason': log.fail_reason, + 'login_time': log.login_time.strftime('%Y-%m-%d %H:%M:%S') if log.login_time else None + } + + return JsonResponse({ + 'code': 200, + 'message': '获取登录日志详情成功', + 'data': log_data + }) + except Exception as e: + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/notification.py b/backend/apps/views/notification.py new file mode 100644 index 0000000..46f54c6 --- /dev/null +++ b/backend/apps/views/notification.py @@ -0,0 +1,401 @@ +import json +import uuid +import hashlib +import hmac +import base64 +import time +import logging +import requests +from urllib.parse import quote_plus +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.db import transaction +from ..models import NotificationRobot, User +from ..utils.auth import jwt_auth_required + +logger = logging.getLogger('apps') + +def generate_id(): + """生成唯一ID""" + return hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:32] + +@method_decorator(csrf_exempt, name='dispatch') +class NotificationRobotView(View): + @method_decorator(jwt_auth_required) + def get(self, request, robot_id=None): + """获取机器人列表或单个机器人详情""" + try: + if robot_id: + try: + robot = NotificationRobot.objects.select_related('creator').get(robot_id=robot_id) + return JsonResponse({ + 'code': 200, + 'message': '获取机器人详情成功', + 'data': { + 'robot_id': robot.robot_id, + 'type': robot.type, + 'name': robot.name, + 'webhook': robot.webhook, + 'security_type': robot.security_type, + 'secret': robot.secret, + 'keywords': robot.keywords, + 'ip_list': robot.ip_list, + 'remark': robot.remark, + 'creator': { + 'user_id': robot.creator.user_id, + 'name': robot.creator.name + } if robot.creator else None, + 'create_time': robot.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'update_time': robot.update_time.strftime('%Y-%m-%d %H:%M:%S') + } + }) + except NotificationRobot.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '机器人不存在' + }) + + # 获取所有机器人列表 + robots = NotificationRobot.objects.select_related('creator').all() + robot_list = [] + for robot in robots: + robot_list.append({ + 'robot_id': robot.robot_id, + 'type': robot.type, + 'name': robot.name, + 'webhook': robot.webhook, + 'security_type': robot.security_type, + 'secret': robot.secret, + 'keywords': robot.keywords, + 'ip_list': robot.ip_list, + 'remark': robot.remark, + 'creator': { + 'user_id': robot.creator.user_id, + 'name': robot.creator.name + } if robot.creator else None, + 'create_time': robot.create_time.strftime('%Y-%m-%d %H:%M:%S') + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取机器人列表成功', + 'data': robot_list + }) + + except Exception as e: + logger.error(f'获取机器人失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def post(self, request): + """创建机器人""" + try: + data = json.loads(request.body) + robot_type = data.get('type') + name = data.get('name') + webhook = data.get('webhook') + security_type = data.get('security_type', 'none') + secret = data.get('secret') + keywords = data.get('keywords', []) + ip_list = data.get('ip_list', []) + remark = data.get('remark') + + # 验证必要字段 + if not all([robot_type, name, webhook]): + return JsonResponse({ + 'code': 400, + 'message': '机器人类型、名称和Webhook地址不能为空' + }) + + # 验证机器人类型 + if robot_type not in ['dingtalk', 'wecom', 'feishu']: + return JsonResponse({ + 'code': 400, + 'message': '不支持的机器人类型' + }) + + # 验证安全设置 + if security_type == 'secret' and not secret: + return JsonResponse({ + 'code': 400, + 'message': '使用加签密钥时，密钥不能为空' + }) + elif security_type == 'keyword' and not keywords: + return JsonResponse({ + 'code': 400, + 'message': '使用自定义关键词时，关键词不能为空' + }) + elif security_type == 'ip' and not ip_list: + return JsonResponse({ + 'code': 400, + 'message': '使用IP白名单时，IP列表不能为空' + }) + + # 创建机器人 + creator = User.objects.get(user_id=request.user_id) + robot = NotificationRobot.objects.create( + robot_id=generate_id(), + type=robot_type, + name=name, + webhook=webhook, + security_type=security_type, + secret=secret, + keywords=keywords, + ip_list=ip_list, + remark=remark, + creator=creator + ) + + return JsonResponse({ + 'code': 200, + 'message': '创建机器人成功', + 'data': { + 'robot_id': robot.robot_id + } + }) + + except Exception as e: + logger.error(f'创建机器人失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """更新机器人""" + try: + data = json.loads(request.body) + robot_id = data.get('robot_id') + name = data.get('name') + webhook = data.get('webhook') + security_type = data.get('security_type') + secret = data.get('secret') + keywords = data.get('keywords') + ip_list = data.get('ip_list') + remark = data.get('remark') + + if not robot_id: + return JsonResponse({ + 'code': 400, + 'message': '机器人ID不能为空' + }) + + try: + robot = NotificationRobot.objects.get(robot_id=robot_id) + except NotificationRobot.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '机器人不存在' + }) + + # 验证必要字段 + if name: + robot.name = name + if webhook: + robot.webhook = webhook + if security_type: + robot.security_type = security_type + if secret is not None: + robot.secret = secret + if keywords is not None: + robot.keywords = keywords + if ip_list is not None: + robot.ip_list = ip_list + if remark is not None: + robot.remark = remark + + # 验证安全设置 + if robot.security_type == 'secret' and not robot.secret: + return JsonResponse({ + 'code': 400, + 'message': '使用加签密钥时，密钥不能为空' + }) + elif robot.security_type == 'keyword' and not robot.keywords: + return JsonResponse({ + 'code': 400, + 'message': '使用自定义关键词时，关键词不能为空' + }) + elif robot.security_type == 'ip' and not robot.ip_list: + return JsonResponse({ + 'code': 400, + 'message': '使用IP白名单时，IP列表不能为空' + }) + + robot.save() + + return JsonResponse({ + 'code': 200, + 'message': '更新机器人成功' + }) + + except Exception as e: + logger.error(f'更新机器人失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def delete(self, request): + """删除机器人""" + try: + data = json.loads(request.body) + robot_id = data.get('robot_id') + + if not robot_id: + return JsonResponse({ + 'code': 400, + 'message': '机器人ID不能为空' + }) + + try: + robot = NotificationRobot.objects.get(robot_id=robot_id) + robot.delete() + return JsonResponse({ + 'code': 200, + 'message': '删除机器人成功' + }) + except NotificationRobot.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '机器人不存在' + }) + + except Exception as e: + logger.error(f'删除机器人失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + +@method_decorator(csrf_exempt, name='dispatch') +class NotificationTestView(View): + def _sign_dingtalk(self, secret, timestamp): + """钉钉机器人签名""" + string_to_sign = f'{timestamp}\n{secret}' + hmac_code = hmac.new( + secret.encode('utf-8'), + string_to_sign.encode('utf-8'), + digestmod=hashlib.sha256 + ).digest() + return base64.b64encode(hmac_code).decode('utf-8') + + def _sign_feishu(self, secret, timestamp): + """飞书机器人签名""" + string_to_sign = f'{timestamp}\n{secret}' + hmac_code = hmac.new( + secret.encode('utf-8'), + string_to_sign.encode('utf-8'), + digestmod=hashlib.sha256 + ).digest() + return base64.b64encode(hmac_code).decode('utf-8') + + @method_decorator(jwt_auth_required) + def post(self, request, *args, **kwargs): + """测试机器人""" + try: + data = json.loads(request.body) + robot_id = data.get('robot_id') + + if not robot_id: + return JsonResponse({ + 'code': 400, + 'message': '机器人ID不能为空' + }) + + try: + robot = NotificationRobot.objects.get(robot_id=robot_id) + except NotificationRobot.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '机器人不存在' + }) + + # 准备测试消息 + timestamp = str(int(time.time() * 1000)) + test_message = "这是一条测试消息，如果您收到了这条消息，说明机器人配置正确。" + + # 根据不同类型的机器人发送测试消息 + try: + if robot.type == 'dingtalk': + # 钉钉机器人 + webhook = robot.webhook + + # 如果使用加签方式 + if robot.security_type == 'secret' and robot.secret: + sign = self._sign_dingtalk(robot.secret, timestamp) + webhook = f"{webhook}×tamp={timestamp}&sign={quote_plus(sign)}" + + # 构建消息内容 + message_data = { + "msgtype": "text", + "text": { + "content": test_message + } + } + + response = requests.post(webhook, json=message_data) + + elif robot.type == 'wecom': + # 企业微信机器人 + response = requests.post(robot.webhook, json={ + "msgtype": "text", + "text": { + "content": test_message + } + }) + + elif robot.type == 'feishu': + # 飞书机器人 + headers = {} + if robot.security_type == 'secret' and robot.secret: + sign = self._sign_feishu(robot.secret, timestamp) + headers.update({ + "X-Timestamp": timestamp, + "X-Sign": sign + }) + + response = requests.post(robot.webhook, json={ + "msg_type": "text", + "content": { + "text": test_message + } + }, headers=headers) + + if response.status_code == 200: + resp_json = response.json() + if resp_json.get('errcode') == 0 or resp_json.get('StatusCode') == 0 or resp_json.get('code') == 0: + return JsonResponse({ + 'code': 200, + 'message': '测试消息发送成功' + }) + else: + return JsonResponse({ + 'code': 400, + 'message': f'测试消息发送失败: {response.text}' + }) + else: + return JsonResponse({ + 'code': 400, + 'message': f'测试消息发送失败: {response.text}' + }) + + except Exception as e: + logger.error(f'发送测试消息失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'发送测试消息失败: {str(e)}' + }) + + except Exception as e: + logger.error(f'测试机器人失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/project.py b/backend/apps/views/project.py new file mode 100644 index 0000000..125282b --- /dev/null +++ b/backend/apps/views/project.py @@ -0,0 +1,392 @@ +import json +import uuid +import hashlib +import logging +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.db import transaction +from django.db.models import Q +from ..models import Project, User +from ..utils.auth import jwt_auth_required +from ..utils.permissions import get_user_permissions + +logger = logging.getLogger('apps') + +def generate_id(): + """生成唯一ID""" + return hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:32] + +@method_decorator(csrf_exempt, name='dispatch') +class ProjectView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取项目列表""" + try: + # 获取当前用户的权限信息 + user_permissions = get_user_permissions(request.user_id) + data_permissions = user_permissions.get('data', {}) + + # 检查用户是否有项目查看权限 + function_permissions = user_permissions.get('function', {}) + project_permissions = function_permissions.get('project', []) + + if 'view' not in project_permissions: + logger.warning(f'用户[{request.user_id}]没有项目查看权限') + return JsonResponse({ + 'code': 403, + 'message': '没有权限查看项目' + }, status=403) + + project_id = request.GET.get('project_id') + name = request.GET.get('name') + category = request.GET.get('category') + + # 构建查询条件 + query = Q() + + if project_id: + query &= Q(project_id=project_id) + if name: + query &= Q(name__icontains=name) # 使用 icontains 进行不区分大小写的模糊查询 + if category: + query &= Q(category=category) + + # 应用项目权限过滤 + project_scope = data_permissions.get('project_scope', 'all') + if project_scope == 'custom': + permitted_project_ids = data_permissions.get('project_ids', []) + if not permitted_project_ids: + # 如果设置了自定义项目权限但列表为空，意味着没有权限查看任何项目 + logger.info(f'用户[{request.user_id}]没有权限查看任何项目') + return JsonResponse({ + 'code': 200, + 'message': '获取项目列表成功', + 'data': [] + }) + + # 限制只能查看有权限的项目 + query &= Q(project_id__in=permitted_project_ids) + + # 使用查询条件过滤项目 + projects = Project.objects.select_related('creator').filter(query) + + project_list = [] + for project in projects: + project_list.append({ + 'project_id': project.project_id, + 'name': project.name, + 'description': project.description, + 'category': project.category, + 'repository': project.repository, + 'creator': { + 'user_id': project.creator.user_id, + 'username': project.creator.username, + 'name': project.creator.name + }, + 'create_time': project.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'update_time': project.update_time.strftime('%Y-%m-%d %H:%M:%S') + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取项目列表成功', + 'data': project_list + }) + except Exception as e: + logger.error(f'获取项目列表失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def post(self, request): + """创建项目""" + try: + with transaction.atomic(): + data = json.loads(request.body) + name = data.get('name') + description = data.get('description') + category = data.get('category') + repository = data.get('repository') + + if not name: + return JsonResponse({ + 'code': 400, + 'message': '项目名称不能为空' + }) + + # 检查项目名称是否已存在 + if Project.objects.filter(name=name).exists(): + return JsonResponse({ + 'code': 400, + 'message': '项目名称已存在' + }) + + # 创建项目 + creator = User.objects.get(user_id=request.user_id) + project = Project.objects.create( + project_id=generate_id(), + name=name, + description=description, + category=category, + repository=repository, + creator=creator + ) + + return JsonResponse({ + 'code': 200, + 'message': '创建项目成功', + 'data': { + 'project_id': project.project_id, + 'name': project.name + } + }) + except Exception as e: + logger.error(f'创建项目失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """编辑项目""" + try: + with transaction.atomic(): + data = json.loads(request.body) + project_id = data.get('project_id') + name = data.get('name') + description = data.get('description') + category = data.get('category') + repository = data.get('repository') + + if not project_id: + return JsonResponse({ + 'code': 400, + 'message': '项目ID不能为空' + }) + + try: + project = Project.objects.get(project_id=project_id) + except Project.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '项目不存在' + }) + + # 检查项目名称是否已存在 + if name and name != project.name: + if Project.objects.filter(name=name).exists(): + return JsonResponse({ + 'code': 400, + 'message': '项目名称已存在' + }) + project.name = name + + if description is not None: + project.description = description + if category: + project.category = category + if repository: + project.repository = repository + + project.save() + + return JsonResponse({ + 'code': 200, + 'message': '更新项目成功' + }) + except Exception as e: + logger.error(f'更新项目失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def delete(self, request): + """删除项目""" + try: + with transaction.atomic(): + data = json.loads(request.body) + project_id = data.get('project_id') + + if not project_id: + return JsonResponse({ + 'code': 400, + 'message': '项目ID不能为空' + }) + + try: + project = Project.objects.get(project_id=project_id) + project.delete() + return JsonResponse({ + 'code': 200, + 'message': '删除项目成功' + }) + except Project.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '项目不存在' + }) + + except Exception as e: + logger.error(f'删除项目失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + +@method_decorator(csrf_exempt, name='dispatch') +class ProjectServiceView(View): + @method_decorator(jwt_auth_required) + def post(self, request): + """创建项目服务""" + try: + with transaction.atomic(): + data = json.loads(request.body) + project_id = data.get('project_id') + name = data.get('name') + description = data.get('description') + category = data.get('category') + repository = data.get('repository') + + if not all([project_id, name, category, repository]): + return JsonResponse({ + 'code': 400, + 'message': '项目ID、服务名称、类别和仓库地址不能为空' + }) + + try: + project = Project.objects.get(project_id=project_id) + except Project.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '项目不存在' + }) + + # 检查服务名称在项目下是否已存在 + if ProjectService.objects.filter(project=project, name=name).exists(): + return JsonResponse({ + 'code': 400, + 'message': '该项目下已存在同名服务' + }) + + creator = User.objects.get(user_id=request.user_id) + service = ProjectService.objects.create( + service_id=generate_id(), + project=project, + name=name, + description=description, + category=category, + repository=repository, + creator=creator + ) + + return JsonResponse({ + 'code': 200, + 'message': '创建服务成功', + 'data': { + 'service_id': service.service_id, + 'name': service.name + } + }) + except Exception as e: + logger.error(f'创建服务失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """更新项目服务""" + try: + with transaction.atomic(): + data = json.loads(request.body) + service_id = data.get('service_id') + name = data.get('name') + description = data.get('description') + category = data.get('category') + repository = data.get('repository') + + if not service_id: + return JsonResponse({ + 'code': 400, + 'message': '服务ID不能为空' + }) + + try: + service = ProjectService.objects.get(service_id=service_id) + except ProjectService.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '服务不存在' + }) + + # 检查服务名称是否已存在 + if name and name != service.name: + if ProjectService.objects.filter(project=service.project, name=name).exists(): + return JsonResponse({ + 'code': 400, + 'message': '该项目下已存在同名服务' + }) + service.name = name + + if description is not None: + service.description = description + if category: + service.category = category + if repository: + service.repository = repository + + service.save() + + return JsonResponse({ + 'code': 200, + 'message': '更新服务成功' + }) + except Exception as e: + logger.error(f'更新服务失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def delete(self, request): + """删除项目服务""" + try: + with transaction.atomic(): + data = json.loads(request.body) + service_id = data.get('service_id') + + if not service_id: + return JsonResponse({ + 'code': 400, + 'message': '服务ID不能为空' + }) + + try: + service = ProjectService.objects.get(service_id=service_id) + service.delete() + return JsonResponse({ + 'code': 200, + 'message': '删除服务成功' + }) + except ProjectService.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '服务不存在' + }) + + except Exception as e: + logger.error(f'删除服务失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/role.py b/backend/apps/views/role.py new file mode 100644 index 0000000..568854b --- /dev/null +++ b/backend/apps/views/role.py @@ -0,0 +1,346 @@ +import json +import uuid +import hashlib +import logging +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.db import transaction +from django.db.models import Q +from ..models import User, Role, UserRole +from ..utils.auth import jwt_auth_required + +logger = logging.getLogger('apps') + +def generate_id(): + """生成唯一ID""" + return hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:32] + +@method_decorator(csrf_exempt, name='dispatch') +class RoleView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取角色列表""" + try: + role_id = request.GET.get('role_id') + name = request.GET.get('name') + + # 构建查询条件 + query = {} + + if role_id: + query['role_id'] = role_id + if name: + query['name__icontains'] = name + + # 使用查询条件过滤角色 + roles = Role.objects.select_related('creator').filter(**query) + + role_list = [] + for role in roles: + # 处理permissions字段，确保是对象形式 + permissions = role.permissions + if isinstance(permissions, str): + try: + permissions = json.loads(permissions) + except json.JSONDecodeError: + logger.error(f'角色[{role.name}]的权限数据格式错误') + permissions = {} + + role_data = { + 'role_id': role.role_id, + 'name': role.name, + 'description': role.description, + 'permissions': permissions, # 返回处理后的权限配置 + 'creator': { + 'user_id': role.creator.user_id, + 'username': role.creator.username, + 'name': role.creator.name + } if role.creator else None, + 'create_time': role.create_time.strftime('%Y-%m-%d %H:%M:%S'), + 'update_time': role.update_time.strftime('%Y-%m-%d %H:%M:%S'), + } + role_list.append(role_data) + + return JsonResponse({ + 'code': 200, + 'message': '获取角色列表成功', + 'data': role_list + }) + except Exception as e: + logger.error(f'获取角色列表失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def post(self, request): + """创建角色""" + try: + with transaction.atomic(): + data = json.loads(request.body) + name = data.get('name') + description = data.get('description') + permissions = data.get('permissions', {}) + + if not name: + return JsonResponse({ + 'code': 400, + 'message': '角色名称不能为空' + }) + + # 检查角色名称是否已存在 + if Role.objects.filter(name=name).exists(): + return JsonResponse({ + 'code': 400, + 'message': '角色名称已存在' + }) + + # 创建角色 + creator = User.objects.get(user_id=request.user_id) + role = Role.objects.create( + role_id=generate_id(), + name=name, + description=description, + permissions=permissions, + creator=creator + ) + + return JsonResponse({ + 'code': 200, + 'message': '创建角色成功', + 'data': { + 'role_id': role.role_id, + 'name': role.name + } + }) + except Exception as e: + logger.error(f'创建角色失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """编辑角色""" + try: + with transaction.atomic(): + data = json.loads(request.body) + role_id = data.get('role_id') + name = data.get('name') + description = data.get('description') + permissions = data.get('permissions') + + if not role_id: + return JsonResponse({ + 'code': 400, + 'message': '角色ID不能为空' + }) + + try: + role = Role.objects.get(role_id=role_id) + except Role.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '角色不存在' + }) + + # 检查角色名称是否已存在 + if name and name != role.name: + if Role.objects.filter(name=name).exists(): + return JsonResponse({ + 'code': 400, + 'message': '角色名称已存在' + }) + role.name = name + + if description is not None: + role.description = description + + # 处理permissions字段 + if permissions is not None: + # 确保permissions是对象而不是字符串 + if isinstance(permissions, str): + try: + permissions = json.loads(permissions) + except json.JSONDecodeError: + return JsonResponse({ + 'code': 400, + 'message': '权限数据格式错误' + }) + role.permissions = permissions + + role.save() + + return JsonResponse({ + 'code': 200, + 'message': '更新角色成功' + }) + except Exception as e: + logger.error(f'更新角色失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def delete(self, request): + """删除角色""" + try: + with transaction.atomic(): + data = json.loads(request.body) + role_id = data.get('role_id') + + if not role_id: + return JsonResponse({ + 'code': 400, + 'message': '角色ID不能为空' + }) + + try: + role = Role.objects.get(role_id=role_id) + + # 检查是否有用户使用该角色 + if UserRole.objects.filter(role=role).exists(): + return JsonResponse({ + 'code': 400, + 'message': '该角色已分配给用户，无法删除' + }) + + role.delete() + return JsonResponse({ + 'code': 200, + 'message': '删除角色成功' + }) + except Role.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '角色不存在' + }) + + except Exception as e: + logger.error(f'删除角色失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + +@method_decorator(csrf_exempt, name='dispatch') +class UserPermissionView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取当前用户的权限""" + try: + user_id = request.user_id + try: + user = User.objects.get(user_id=user_id) + except User.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '用户不存在' + }) + + # 记录操作日志 + logger.info(f'用户[{user.username}]获取权限信息') + + # 获取用户的所有角色 + user_roles = UserRole.objects.filter(user=user).select_related('role') + + # 合并所有角色的权限 + menu_permissions = set() + function_permissions = {} + data_permissions = { + 'project_scope': 'all', + 'project_ids': [], + 'environment_scope': 'all', + 'environment_types': [] + } + + has_custom_project_scope = False + has_custom_environment_scope = False + + for user_role in user_roles: + role = user_role.role + permissions = role.permissions + + # 如果permissions是字符串，解析为JSON + if isinstance(permissions, str): + try: + permissions = json.loads(permissions) + except json.JSONDecodeError: + logger.error(f'解析角色[{role.name}]的权限数据失败') + permissions = {} + + # 合并菜单权限 + if permissions.get('menu') and isinstance(permissions['menu'], list): + menu_permissions.update(permissions['menu']) + + # 合并功能权限 + if permissions.get('function') and isinstance(permissions['function'], dict): + for module, actions in permissions['function'].items(): + if not isinstance(actions, list): + continue + + if module not in function_permissions: + function_permissions[module] = [] + function_permissions[module].extend(actions) + # 确保不重复 + function_permissions[module] = list(set(function_permissions[module])) + + # 合并数据权限 + if permissions.get('data') and isinstance(permissions['data'], dict): + data_perms = permissions['data'] + + # 项目权限 + if data_perms.get('project_scope') == 'custom': + has_custom_project_scope = True + if data_permissions['project_scope'] == 'all': + data_permissions['project_scope'] = 'custom' + data_permissions['project_ids'] = data_perms.get('project_ids', []) + else: + # 合并项目ID列表 + data_permissions['project_ids'].extend(data_perms.get('project_ids', [])) + # 确保不重复 + data_permissions['project_ids'] = list(set(data_permissions['project_ids'])) + + # 环境权限 + if data_perms.get('environment_scope') == 'custom': + has_custom_environment_scope = True + if data_permissions['environment_scope'] == 'all': + data_permissions['environment_scope'] = 'custom' + data_permissions['environment_types'] = data_perms.get('environment_types', []) + else: + # 合并环境类型列表 + data_permissions['environment_types'].extend(data_perms.get('environment_types', [])) + # 确保不重复 + data_permissions['environment_types'] = list(set(data_permissions['environment_types'])) + + # 如果没有任何角色有自定义项目/环境范围，保持为'all' + if not has_custom_project_scope: + data_permissions['project_scope'] = 'all' + data_permissions['project_ids'] = [] + + if not has_custom_environment_scope: + data_permissions['environment_scope'] = 'all' + data_permissions['environment_types'] = [] + + permissions_result = { + 'menu': list(menu_permissions), + 'function': function_permissions, + 'data': data_permissions + } + + return JsonResponse({ + 'code': 200, + 'message': '获取用户权限成功', + 'data': permissions_result + }) + except Exception as e: + logger.error(f'获取用户权限失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/security.py b/backend/apps/views/security.py new file mode 100644 index 0000000..b0f6d19 --- /dev/null +++ b/backend/apps/views/security.py @@ -0,0 +1,143 @@ +import json +import logging +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.db import transaction +from ..models import SecurityConfig, User +from ..utils.auth import jwt_auth_required + +logger = logging.getLogger('apps') + +@method_decorator(csrf_exempt, name='dispatch') +class SecurityConfigView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取安全配置""" + try: + # 获取或创建安全配置 + security_config, created = SecurityConfig.objects.get_or_create( + id=1, + defaults={ + 'min_password_length': 8, + 'password_complexity': ['lowercase', 'number'], + 'session_timeout': 120, + 'max_login_attempts': 5, + 'lockout_duration': 30, + 'enable_2fa': False + } + ) + + return JsonResponse({ + 'code': 200, + 'message': '获取安全配置成功', + 'data': { + 'min_password_length': security_config.min_password_length, + 'password_complexity': security_config.password_complexity, + 'session_timeout': security_config.session_timeout, + 'max_login_attempts': security_config.max_login_attempts, + 'lockout_duration': security_config.lockout_duration, + 'enable_2fa': security_config.enable_2fa, + 'update_time': security_config.update_time.strftime('%Y-%m-%d %H:%M:%S') if security_config.update_time else None + } + }) + + except Exception as e: + logger.error(f'获取安全配置失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """更新安全配置""" + try: + with transaction.atomic(): + data = json.loads(request.body) + + min_password_length = data.get('min_password_length') + password_complexity = data.get('password_complexity') + session_timeout = data.get('session_timeout') + max_login_attempts = data.get('max_login_attempts') + lockout_duration = data.get('lockout_duration') + enable_2fa = data.get('enable_2fa') + + # 验证输入数据 + if min_password_length is not None: + if not isinstance(min_password_length, int) or min_password_length < 6 or min_password_length > 20: + return JsonResponse({ + 'code': 400, + 'message': '密码最小长度必须在6-20之间' + }) + + if password_complexity is not None: + if not isinstance(password_complexity, list): + return JsonResponse({ + 'code': 400, + 'message': '密码复杂度要求格式错误' + }) + valid_complexity = ['uppercase', 'lowercase', 'number', 'special'] + for item in password_complexity: + if item not in valid_complexity: + return JsonResponse({ + 'code': 400, + 'message': f'无效的密码复杂度要求: {item}' + }) + + if session_timeout is not None: + if not isinstance(session_timeout, int) or session_timeout < 10 or session_timeout > 1440: + return JsonResponse({ + 'code': 400, + 'message': '会话超时时间必须在10-1440分钟之间' + }) + + if max_login_attempts is not None: + if not isinstance(max_login_attempts, int) or max_login_attempts < 3 or max_login_attempts > 10: + return JsonResponse({ + 'code': 400, + 'message': '最大登录尝试次数必须在3-10次之间' + }) + + if lockout_duration is not None: + if not isinstance(lockout_duration, int) or lockout_duration < 5 or lockout_duration > 60: + return JsonResponse({ + 'code': 400, + 'message': '账户锁定时间必须在5-60分钟之间' + }) + + # 获取或创建安全配置 + security_config, created = SecurityConfig.objects.get_or_create(id=1) + + # 更新配置 + if min_password_length is not None: + security_config.min_password_length = min_password_length + if password_complexity is not None: + security_config.password_complexity = password_complexity + if session_timeout is not None: + security_config.session_timeout = session_timeout + if max_login_attempts is not None: + security_config.max_login_attempts = max_login_attempts + if lockout_duration is not None: + security_config.lockout_duration = lockout_duration + if enable_2fa is not None: + security_config.enable_2fa = enable_2fa + + security_config.save() + + # 记录操作日志 + user = User.objects.get(user_id=request.user_id) + logger.info(f'用户[{user.username}]更新了安全配置') + + return JsonResponse({ + 'code': 200, + 'message': '安全配置更新成功' + }) + + except Exception as e: + logger.error(f'更新安全配置失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/apps/views/user.py b/backend/apps/views/user.py new file mode 100644 index 0000000..1bd5d87 --- /dev/null +++ b/backend/apps/views/user.py @@ -0,0 +1,334 @@ +import json +import uuid +import hashlib +import logging +from django.http import JsonResponse +from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.db import transaction +from django.db.models import Q +from ..models import User, Role, UserRole +from ..utils.auth import jwt_auth_required +from ..utils.security import SecurityValidator + +logger = logging.getLogger('apps') + +def generate_id(): + """生成唯一ID""" + return hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:32] + +@method_decorator(csrf_exempt, name='dispatch') +class UserView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取用户列表""" + try: + user_id = request.GET.get('user_id') + username = request.GET.get('username') + email = request.GET.get('email') + status = request.GET.get('status') + + # 构建查询条件 + query = {} + + if user_id: + query['user_id'] = user_id + if username: + query['username__icontains'] = username # 使用 icontains 进行不区分大小写的模糊查询 + if email: + query['email__icontains'] = email + if status: + query['status'] = status + + # 使用查询条件过滤用户 + users = User.objects.filter(**query) + + user_list = [] + for user in users: + # 获取用户角色 + user_roles = UserRole.objects.filter(user=user).select_related('role') + roles = [{"role_id": ur.role.role_id, "name": ur.role.name} for ur in user_roles] + + user_list.append({ + 'user_id': user.user_id, + 'username': user.username, + 'name': user.name, + 'email': user.email, + 'status': user.status, + 'roles': roles, + 'login_time': user.login_time.strftime('%Y-%m-%d %H:%M:%S') if user.login_time else None, + 'create_time': user.create_time.strftime('%Y-%m-%d %H:%M:%S'), + }) + + return JsonResponse({ + 'code': 200, + 'message': '获取用户列表成功', + 'data': user_list + }) + except Exception as e: + logger.error(f'获取用户列表失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def post(self, request): + """创建用户""" + try: + with transaction.atomic(): + data = json.loads(request.body) + username = data.get('username') + name = data.get('name') + password = data.get('password') + email = data.get('email') + role_ids = data.get('role_ids', []) + + if not all([username, name, password, email]): + return JsonResponse({ + 'code': 400, + 'message': '用户名、姓名、密码和邮箱不能为空' + }) + + # 检查用户名是否已存在 + if User.objects.filter(username=username).exists(): + return JsonResponse({ + 'code': 400, + 'message': '用户名已存在' + }) + + # 检查邮箱是否已存在 + if User.objects.filter(email=email).exists(): + return JsonResponse({ + 'code': 400, + 'message': '邮箱已存在' + }) + + # 验证密码强度 + is_valid, message = SecurityValidator.validate_password(password) + if not is_valid: + return JsonResponse({ + 'code': 400, + 'message': message + }) + + # 密码加密 + password_hash = hashlib.sha256(password.encode()).hexdigest() + + # 创建用户 + user = User.objects.create( + user_id=generate_id(), + username=username, + name=name, + password=password_hash, + email=email, + status=1 # 默认启用 + ) + + # 分配角色 + for role_id in role_ids: + try: + role = Role.objects.get(role_id=role_id) + UserRole.objects.create(user=user, role=role) + except Role.DoesNotExist: + logger.warning(f'角色不存在: {role_id}') + + return JsonResponse({ + 'code': 200, + 'message': '创建用户成功', + 'data': { + 'user_id': user.user_id, + 'username': user.username + } + }) + except Exception as e: + logger.error(f'创建用户失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def put(self, request): + """编辑用户""" + try: + with transaction.atomic(): + data = json.loads(request.body) + user_id = data.get('user_id') + name = data.get('name') + email = data.get('email') + status = data.get('status') + password = data.get('password') + role_ids = data.get('role_ids') + + if not user_id: + return JsonResponse({ + 'code': 400, + 'message': '用户ID不能为空' + }) + + try: + user = User.objects.get(user_id=user_id) + except User.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '用户不存在' + }) + + # 更新用户信息 + if name: + user.name = name + if email and email != user.email: + # 检查邮箱是否已存在 + if User.objects.filter(email=email).exclude(user_id=user_id).exists(): + return JsonResponse({ + 'code': 400, + 'message': '邮箱已存在' + }) + user.email = email + if status is not None: + # 如果是解锁操作（从0改为1），使用安全验证器的解锁方法 + if user.status == 0 and status == 1: + success, message = SecurityValidator.unlock_user_account(user) + if not success: + return JsonResponse({ + 'code': 400, + 'message': message + }) + # 如果是锁定操作（从1改为0），使用安全验证器的锁定方法 + elif user.status == 1 and status == 0: + success, message = SecurityValidator.lock_user_account(user) + if not success: + return JsonResponse({ + 'code': 400, + 'message': message + }) + else: + user.status = status + if password: + # 验证密码强度 + is_valid, message = SecurityValidator.validate_password(password) + if not is_valid: + return JsonResponse({ + 'code': 400, + 'message': message + }) + + # 密码加密 + password_hash = hashlib.sha256(password.encode()).hexdigest() + user.password = password_hash + + user.save() + + # 更新角色 + if role_ids is not None: + # 删除旧角色关联 + UserRole.objects.filter(user=user).delete() + + # 添加新角色关联 + for role_id in role_ids: + try: + role = Role.objects.get(role_id=role_id) + UserRole.objects.create(user=user, role=role) + except Role.DoesNotExist: + logger.warning(f'角色不存在: {role_id}') + + return JsonResponse({ + 'code': 200, + 'message': '更新用户成功' + }) + except Exception as e: + logger.error(f'更新用户失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + @method_decorator(jwt_auth_required) + def delete(self, request): + """删除用户""" + try: + with transaction.atomic(): + data = json.loads(request.body) + user_id = data.get('user_id') + + if not user_id: + return JsonResponse({ + 'code': 400, + 'message': '用户ID不能为空' + }) + + try: + user = User.objects.get(user_id=user_id) + # 删除关联的角色 + UserRole.objects.filter(user=user).delete() + # 删除用户 + user.delete() + return JsonResponse({ + 'code': 200, + 'message': '删除用户成功' + }) + except User.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '用户不存在' + }) + + except Exception as e: + logger.error(f'删除用户失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) + + +@method_decorator(csrf_exempt, name='dispatch') +class UserProfileView(View): + @method_decorator(jwt_auth_required) + def get(self, request): + """获取当前登录用户的个人信息""" + try: + user_id = request.user_id + + try: + user = User.objects.get(user_id=user_id) + except User.DoesNotExist: + return JsonResponse({ + 'code': 404, + 'message': '用户不存在' + }) + + # 获取用户角色 + user_roles = UserRole.objects.filter(user=user).select_related('role') + roles = [{ + "role_id": ur.role.role_id, + "name": ur.role.name, + "description": ur.role.description + } for ur in user_roles] + + # 构建用户信息 + user_info = { + 'user_id': user.user_id, + 'username': user.username, + 'name': user.name, + 'email': user.email, + 'status': user.status, + 'roles': roles, + 'login_time': user.login_time.strftime('%Y-%m-%d %H:%M:%S') if user.login_time else None, + 'create_time': user.create_time.strftime('%Y-%m-%d %H:%M:%S') if user.create_time else None, + 'update_time': user.update_time.strftime('%Y-%m-%d %H:%M:%S') if user.update_time else None, + } + + return JsonResponse({ + 'code': 200, + 'message': '获取用户信息成功', + 'data': user_info + }) + + except Exception as e: + logger.error(f'获取用户信息失败: {str(e)}', exc_info=True) + return JsonResponse({ + 'code': 500, + 'message': f'服务器错误: {str(e)}' + }) \ No newline at end of file diff --git a/backend/backend/__init__.py b/backend/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/backend/asgi.py b/backend/backend/asgi.py new file mode 100644 index 0000000..420897a --- /dev/null +++ b/backend/backend/asgi.py @@ -0,0 +1,14 @@ +""" +ASGI config for backend project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" +import os +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings') + +application = get_asgi_application() \ No newline at end of file diff --git a/backend/backend/settings.py b/backend/backend/settings.py new file mode 100644 index 0000000..f1188ed --- /dev/null +++ b/backend/backend/settings.py @@ -0,0 +1,199 @@ +from pathlib import Path +import pymysql +import logging + +BASE_DIR = Path(__file__).resolve().parent.parent + +SECRET_KEY = 'django-insecure-5z@^6hpk^zxffj_7)&l3pvww8@ky3qsai4)m!vcog!8#=@a3&%' + +DEBUG = True + +ALLOWED_HOSTS = ['*'] # 允许所有主机访问 + +# CORS设置 +CORS_ALLOW_CREDENTIALS = True +CORS_ALLOW_ALL_ORIGINS = True # 允许所有来源 + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'corsheaders', # 添加 cors-headers + 'apps', +] + +MIDDLEWARE = [ + 'corsheaders.middleware.CorsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'backend.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'backend.wsgi.application' + +pymysql.install_as_MySQLdb() + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'OPTIONS': { + 'read_default_file': str(BASE_DIR / 'conf' / 'config.txt'), + 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", + }, + } +} + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +# 基础设置，包括语言、时区等 +LANGUAGE_CODE = 'en-us' # 修改为中文 +TIME_ZONE = 'Asia/Shanghai' +USE_I18N = True +USE_L10N = True +USE_TZ = False # 修改为 True，使用时区功能 + +# 日志配置 +LOG_DIR = BASE_DIR / 'logs' +LOG_DIR.mkdir(exist_ok=True) # 确保日志目录存在 + +class BuildLogFilter(logging.Filter): + def filter(self, record): + # 如果是构建日志，使用简单格式 + if getattr(record, 'from_builder', False): + return True + # 其他日志使用默认格式 + return True + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'build_log_filter': { + '()': 'backend.settings.BuildLogFilter', + }, + 'non_build_log_filter': { + '()': lambda: type('Filter', (logging.Filter,), { + 'filter': lambda self, record: not getattr(record, 'from_builder', False) + })(), + }, + }, + 'formatters': { + 'verbose': { + 'format': '[{asctime}] {levelname} [{name}:{lineno}] {message}', + 'style': '{', + 'datefmt': '%Y-%m-%d %H:%M:%S' + }, + 'simple': { + 'format': '{message}', + 'style': '{' + }, + 'console_with_time': { + 'format': '[{asctime}] {levelname} {message}', + 'style': '{', + 'datefmt': '%Y-%m-%d %H:%M:%S' + }, + }, + 'handlers': { + 'console_build': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple', + 'filters': ['build_log_filter'], + }, + 'console_normal': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'console_with_time', + 'filters': ['non_build_log_filter'], + }, + 'file': { + 'level': 'INFO', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': str(LOG_DIR / 'django.log'), + 'maxBytes': 1024 * 1024 * 5, # 5MB + 'backupCount': 5, + 'formatter': 'verbose', + }, + 'error_file': { + 'level': 'ERROR', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': str(LOG_DIR / 'error.log'), + 'maxBytes': 1024 * 1024 * 5, # 5MB + 'backupCount': 5, + 'formatter': 'verbose', + } + }, + 'loggers': { + 'django': { + 'handlers': ['console_normal', 'file'], + 'level': 'INFO', + 'propagate': True, + }, + 'django.request': { + 'handlers': ['error_file'], + 'level': 'ERROR', + 'propagate': False, + }, + 'django.server': { + 'handlers': ['console_normal', 'file'], + 'level': 'INFO', + 'propagate': False, + }, + 'apps': { + 'handlers': ['console_build', 'console_normal', 'file', 'error_file'], + 'level': 'DEBUG', + 'propagate': False, # 设置为False以避免重复记录 + }, + }, + 'root': { + 'handlers': ['console_normal', 'file', 'error_file'], + 'level': 'INFO', + }, +} + +STATIC_URL = 'static/' + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +# 构建相关配置 +#BUILD_ROOT = Path('/data/liteops/build') # 修改为指定目录 +BUILD_ROOT = Path('/Users/huk/Downloads/data') +BUILD_ROOT.mkdir(exist_ok=True, parents=True) # 确保目录存在，包括父目录 \ No newline at end of file diff --git a/backend/backend/urls.py b/backend/backend/urls.py new file mode 100644 index 0000000..527503f --- /dev/null +++ b/backend/backend/urls.py @@ -0,0 +1,67 @@ +from django.contrib import admin +from django.urls import path +from apps.views.login import login, logout +from apps.views.project import ProjectView, ProjectServiceView +from apps.views.environment import EnvironmentView, EnvironmentTypeView +from apps.views.credentials import CredentialView +from apps.views.gitlab import GitlabBranchView, GitlabCommitView +from apps.views.build import BuildTaskView, BuildExecuteView +from apps.views.build_history import BuildHistoryView, BuildLogView, BuildStageLogView +from apps.views.build_sse import BuildLogSSEView +from apps.views.notification import NotificationRobotView, NotificationTestView +from apps.views.user import UserView, UserProfileView +from apps.views.role import RoleView, UserPermissionView +from apps.views.logs import login_logs_list, login_log_detail +from apps.views.dashboard import DashboardStatsView, BuildTrendView, BuildDetailView, RecentBuildsView, ProjectDistributionView + +from apps.views.security import SecurityConfigView + +urlpatterns = [ + path('admin/', admin.site.urls), + path('api/login/', login, name='login'), + path('api/logout/', logout, name='logout'), + path('api/projects/', ProjectView.as_view(), name='projects'), + path('api/project-services/', ProjectServiceView.as_view(), name='project-services'), + path('api/environments/', EnvironmentView.as_view(), name='environments'), + path('api/environments/types/', EnvironmentTypeView.as_view(), name='environment-types'), + path('api/credentials/', CredentialView.as_view(), name='credentials'), + path('api/gitlab/branches/', GitlabBranchView.as_view(), name='gitlab-branches'), + path('api/gitlab/commits/', GitlabCommitView.as_view(), name='gitlab-commits'), + path('api/build/tasks/', BuildTaskView.as_view(), name='build-tasks'), + path('api/build/tasks//', BuildTaskView.as_view(), name='build-task-detail'), + path('api/build/tasks/build', BuildExecuteView.as_view(), name='build-execute'), + + # 构建历史相关路由 + path('api/build/history/', BuildHistoryView.as_view(), name='build-history'), + path('api/build/history/log//', BuildLogView.as_view(), name='build-log'), + path('api/build/history/log//download/', BuildLogView.as_view(), name='build-log-download'), + path('api/build/history/stage-log///', BuildStageLogView.as_view(), name='build-stage-log'), + + # SSE构建日志流 + path('api/build/logs/stream///', BuildLogSSEView.as_view(), name='build-log-sse'), + + # 通知机器人相关路由 + path('api/notification/robots/', NotificationRobotView.as_view(), name='notification-robots'), + path('api/notification/robots/test/', NotificationTestView.as_view(), name='notification-robot-test'), + path('api/notification/robots//', NotificationRobotView.as_view(), name='notification-robot-detail'), + + # 用户管理相关路由 + path('api/users/', UserView.as_view(), name='users'), + path('api/roles/', RoleView.as_view(), name='roles'), + path('api/user/permissions/', UserPermissionView.as_view(), name='user-permissions'), + path('api/user/profile/', UserProfileView.as_view(), name='user-profile'), + + # 登录日志相关路由 + path('api/logs/login/', login_logs_list, name='login-logs'), + path('api/logs/login//', login_log_detail, name='login-log-detail'), + + # 首页仪表盘相关路由 + path('api/dashboard/stats/', DashboardStatsView.as_view(), name='dashboard-stats'), + path('api/dashboard/build-trend/', BuildTrendView.as_view(), name='build-trend'), + path('api/dashboard/build-detail/', BuildDetailView.as_view(), name='build-detail'), + path('api/dashboard/recent-builds/', RecentBuildsView.as_view(), name='recent-builds'), + path('api/dashboard/project-distribution/', ProjectDistributionView.as_view(), name='project-distribution'), + + # 安全配置相关路由 + path('api/system/security/', SecurityConfigView.as_view(), name='security-config'), +] diff --git a/backend/backend/wsgi.py b/backend/backend/wsgi.py new file mode 100644 index 0000000..21d383b --- /dev/null +++ b/backend/backend/wsgi.py @@ -0,0 +1,7 @@ +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings') + +application = get_wsgi_application() diff --git a/backend/conf/config.txt b/backend/conf/config.txt new file mode 100644 index 0000000..fa7c5e1 --- /dev/null +++ b/backend/conf/config.txt @@ -0,0 +1,8 @@ +[client] +#host = 127.0.0.1 +host = mysql +port = 3306 +database = liteops +user = root +password = 1234567xx +default-character-set = utf8mb4 \ No newline at end of file diff --git a/backend/manage.py b/backend/manage.py new file mode 100755 index 0000000..bd804b0 --- /dev/null +++ b/backend/manage.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + from django.core.management.commands.runserver import Command as Runserver + Runserver.default_addr = '0.0.0.0' # 修改默认地址 + Runserver.default_port = '8900' # 修改默认端口 + main() \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..7d5dc4d --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,12 @@ +asgiref==3.6.0 +# Django==4.2.13 +GitPython==3.1.44 +python-gitlab==5.6.0 +PyJWT==2.10.1 +PyMySQL==1.1.1 +Requests==2.32.3 +uvicorn==0.34.0 +decorator==5.1.1 +django-cors-headers==4.2.0 +cryptography==42.0.5 +PyYAML==6.0.1 \ No newline at end of file diff --git a/ci-entrypoint-dind.sh b/ci-entrypoint-dind.sh new file mode 100644 index 0000000..482a441 --- /dev/null +++ b/ci-entrypoint-dind.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# Docker in Docker (DinD) 启动脚本 - 轻量级CI/CD版本 +# ============================================================================= + +echo "🐳 启动 Docker in Docker 环境..." + +# 检查是否在特权模式下运行 +if [ ! -w /sys/fs/cgroup ]; then + echo "❌ 错误: 容器必须在特权模式下运行才能使用 Docker in Docker" + echo "请使用 --privileged 参数启动容器" + exit 1 +fi + +# 确保必要的内核模块和设备 +modprobe overlay 2>/dev/null || true +modprobe br_netfilter 2>/dev/null || true + +# 创建必要的设备节点 +if [ ! -e /dev/fuse ]; then + mknod /dev/fuse c 10 229 2>/dev/null || true +fi + +# 创建必要的目录 +mkdir -p /var/lib/docker +mkdir -p /var/run/docker +mkdir -p /etc/docker + +# 配置轻量级Docker daemon - 使用vfs存储驱动确保兼容性 +cat > /etc/docker/daemon.json << 'EOF' +{ + "storage-driver": "vfs", + "log-driver": "json-file", + "log-opts": { + "max-size": "10m", + "max-file": "2" + }, + "registry-mirrors": [ + "https://mirrors.aliyun.com/docker-hub", + "https://docker.mirrors.ustc.edu.cn", + "https://hub-mirror.c.163.com" + ], + "insecure-registries": [], + "exec-opt": ["native.cgroupdriver=cgroupfs"], + "max-concurrent-downloads": 3, + "max-concurrent-uploads": 3 +} +EOF + +# 启动轻量级Docker daemon +echo "🚀 启动 Docker daemon (轻量级CI/CD模式)..." + +# 清理可能存在的旧进程 +pkill dockerd 2>/dev/null || true +rm -f /var/run/docker.sock /var/run/docker.pid 2>/dev/null || true + +# 启动dockerd +dockerd \ + --host=unix:///var/run/docker.sock \ + --userland-proxy=false \ + --experimental=false \ + --live-restore=false \ + --iptables=false \ + --ip-forward=false \ + --pidfile=/var/run/docker.pid \ + --tls=false \ + --log-level=warn & + +# 记录dockerd进程ID +DOCKERD_PID=$! + +# 等待Docker daemon启动 +echo "⏳ 等待 Docker daemon 启动..." +timeout=60 +while [ $timeout -gt 0 ]; do + # 检查socket文件是否存在 + if [ -S /var/run/docker.sock ]; then + # 尝试连接Docker daemon + if docker version >/dev/null 2>&1; then + echo "✅ Docker daemon 启动成功" + break + fi + fi + + # 检查dockerd进程是否还在运行 + if ! kill -0 $DOCKERD_PID 2>/dev/null; then + echo "❌ Docker daemon 进程意外退出" + echo "检查最近的错误日志:" + dmesg | tail -5 2>/dev/null || echo "无法获取系统日志" + exit 1 + fi + + sleep 1 + timeout=$((timeout - 1)) +done + +if [ $timeout -eq 0 ]; then + echo "❌ Docker daemon 启动超时" + echo "检查dockerd进程状态:" + ps aux | grep dockerd || true + echo "检查socket文件:" + ls -la /var/run/docker.sock 2>/dev/null || echo "socket文件不存在" + exit 1 +fi + +# 简单验证Docker功能 +echo "🔍 验证 Docker 功能..." +DOCKER_VERSION=$(docker version --format '{{.Server.Version}}' 2>/dev/null) +if [ $? -eq 0 ]; then + echo "✅ Docker daemon 版本: $DOCKER_VERSION" + echo "✅ 存储驱动: $(docker info --format '{{.Driver}}' 2>/dev/null || echo 'unknown')" +else + echo "❌ Docker daemon 验证失败" + exit 1 +fi + +# 设置环境变量 +export DOCKER_HOST=unix:///var/run/docker.sock +export DOCKER_BUILDKIT=1 + +echo "🎉 Docker in Docker 环境启动完成 (轻量级CI/CD模式)" + +# 设置清理函数 +cleanup() { + echo "🧹 清理 Docker daemon..." + if [ -n "$DOCKERD_PID" ] && kill -0 $DOCKERD_PID 2>/dev/null; then + kill $DOCKERD_PID + wait $DOCKERD_PID 2>/dev/null || true + fi + exit 0 +} + +# 注册信号处理 +trap cleanup SIGTERM SIGINT + +# 执行传入的命令 +exec "$@" diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..90929f9 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "Starting nginx..." +service nginx start + +echo "Starting backend service..." +python -m uvicorn backend.asgi:application \ + --host 0.0.0.0 \ + --port 8900 \ + --workers 1 \ No newline at end of file diff --git a/liteops-sidebar.png b/liteops-sidebar.png new file mode 100644 index 0000000..706ab53 Binary files /dev/null and b/liteops-sidebar.png differ diff --git a/start-containers.sh b/start-containers.sh new file mode 100755 index 0000000..9f89991 --- /dev/null +++ b/start-containers.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # 无颜色 + +# 设置变量 +CONTAINER_IMAGE="liteops:v1" +CONTAINER_NAME="liteops" +MYSQL_CONTAINER="liteops-mysql" +MYSQL_VERSION="8" +MYSQL_PASSWORD="1234567xx" +MYSQL_PORT="3306" +NETWORK_NAME="liteops-network" + +# 打印带颜色的信息 +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +print_step() { + echo -e "\n${PURPLE}=== $1 ===${NC}" +} + +# 创建Docker网络（如果不存在） +print_step "创建Docker网络" +if ! docker network inspect $NETWORK_NAME >/dev/null 2>&1; then + print_info "创建Docker网络: $NETWORK_NAME" + docker network create $NETWORK_NAME + print_success "网络创建成功" +else + print_info "网络 $NETWORK_NAME 已存在，跳过创建" +fi + +# 停止并删除已存在的容器 +print_step "清理已存在的容器" +print_info "停止并删除已存在的容器..." +docker stop $CONTAINER_NAME $MYSQL_CONTAINER 2>/dev/null || true +docker rm $CONTAINER_NAME $MYSQL_CONTAINER 2>/dev/null || true +print_success "容器清理完成" + +# 构建镜像 +print_step "构建应用镜像" +print_info "构建LiteOps镜像..." +# 确保前端已经构建 +if [ ! -d "web/dist" ]; then + print_error "前端dist目录不存在，请先运行 npm run build" + exit 1 +fi +docker build --platform linux/amd64 -t $CONTAINER_IMAGE . +print_success "镜像构建成功: $CONTAINER_IMAGE" + +# 启动MySQL容器 +print_step "启动MySQL容器" +print_info "启动MySQL $MYSQL_VERSION 容器..." +docker run -d \ + --name $MYSQL_CONTAINER \ + --network $NETWORK_NAME \ + -p $MYSQL_PORT:3306 \ + -e MYSQL_ROOT_PASSWORD=$MYSQL_PASSWORD \ + -e MYSQL_DATABASE=liteops \ + mysql:$MYSQL_VERSION + +# 等待MySQL启动 +print_info "等待MySQL启动..." +sleep 10 + +# 初始化数据库 +print_step "初始化数据库" +print_info "导入初始数据..." +docker exec -i $MYSQL_CONTAINER mysql -uroot -p$MYSQL_PASSWORD liteops < liteops_init.sql +print_success "数据库初始化完成" + +# 启动应用容器 +print_step "启动应用容器" +print_info "启动LiteOps容器（Docker in Docker模式）..." +docker run -d \ + --name $CONTAINER_NAME \ + --network $NETWORK_NAME \ + --privileged \ + -p 80:80 \ + -p 8900:8900 \ + $CONTAINER_IMAGE + +print_step "部署完成" +print_success "LiteOps已成功部署！" +print_info "前端访问地址: ${CYAN}http://localhost${NC}" +print_info "后端API地址: ${CYAN}http://localhost:8900/api/${NC}" +print_info "MySQL端口映射: ${CYAN}$MYSQL_PORT${NC}" \ No newline at end of file