Merge branch 'dev' into Fix-15570

Author: rickchengx

deploy/kubernetes/dolphinscheduler/templates/configmap.yaml

@@ -32,7 +32,7 @@ data:
       {{- end }}
     {{- end }}
   {{- end }}
-  common_properties: |-
+  common.properties: |-
   {{- if index .Values.conf "common" }}
   {{- range $key, $value := index .Values.conf "common" }}
     {{- if and $.Values.minio.enabled }}

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@@ -114,7 +114,7 @@ spec:
               name: {{ include "dolphinscheduler.fullname" . }}-alert
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
-              subPath: common_properties
+              subPath: common.properties
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-alert
           {{- if .Values.alert.persistentVolumeClaim.enabled }}

deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@@ -115,7 +115,7 @@ spec:
               name: {{ include "dolphinscheduler.fullname" . }}-api
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
-              subPath: common_properties
+              subPath: common.properties
             {{- if .Values.api.taskTypeFilter.enabled }}
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -112,7 +112,7 @@ spec:
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
-              subPath: common_properties
+              subPath: common.properties
             {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}
       volumes:
         - name: {{ include "dolphinscheduler.fullname" . }}-master

deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -113,7 +113,7 @@ spec:
               name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
             - name: config-volume
               mountPath: /opt/dolphinscheduler/conf/common.properties
-              subPath: common_properties
+              subPath: common.properties
             {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
             {{- include "dolphinscheduler.etcd.ssl.volumeMount" . | nindent 12 }}

docs/docs/en/guide/remote-logging.md

@@ -10,7 +10,7 @@ If you deploy DolphinScheduler in `Standalone` mode, you only need to configure
 ```properties
 # Whether to enable remote logging
 remote.logging.enable=false
-# if remote.logging.enable = true, set the target of remote logging
+# if remote.logging.enable = true, set the target of remote logging, currently support OSS, S3, GCS, ABS
 remote.logging.target=OSS
 # if remote.logging.enable = true, set the log base directory
 remote.logging.base.dir=logs
@@ -66,12 +66,12 @@ remote.logging.google.cloud.storage.bucket.name=<your-bucket>
 Configure `common.properties` as follows:
 
 ```properties
-# abs container name, required if you set resource.storage.type=ABS
-resource.azure.blob.storage.container.name=<your-container>
 # abs account name, required if you set resource.storage.type=ABS
-resource.azure.blob.storage.account.name=<your-account-name>
-# abs connection string, required if you set resource.storage.type=ABS
-resource.azure.blob.storage.connection.string=<your-connection-string>
+remote.logging.abs.account.name=<your-account-name>
+# abs account key, required if you set resource.storage.type=ABS
+remote.logging.abs.account.key=<your-account-key>
+# abs container name, required if you set resource.storage.type=ABS
+remote.logging.abs.container.name=<your-container-name>
 ```
 
 ### Notice

docs/docs/zh/guide/remote-logging.md

@@ -10,7 +10,7 @@ Apache DolphinScheduler支持将任务日志传输到远端存储上。当配置
 ```properties
 # 是否开启远程日志存储
 remote.logging.enable=true
-# 任务日志写入的远端存储，目前支持OSS, S3, GCS
+# 任务日志写入的远端存储，目前支持OSS, S3, GCS, ABS
 remote.logging.target=OSS
 # 任务日志在远端存储上的目录
 remote.logging.base.dir=logs
@@ -66,12 +66,12 @@ remote.logging.google.cloud.storage.bucket.name=<your-bucket>
 配置`common.propertis`如下：
 
 ```properties
-# abs container name, required if you set resource.storage.type=ABS
-resource.azure.blob.storage.container.name=<your-container>
 # abs account name, required if you set resource.storage.type=ABS
-resource.azure.blob.storage.account.name=<your-account-name>
-# abs connection string, required if you set resource.storage.type=ABS
-resource.azure.blob.storage.connection.string=<your-connection-string>
+remote.logging.abs.account.name=<your-account-name>
+# abs account key, required if you set resource.storage.type=ABS
+remote.logging.abs.account.key=<your-account-key>
+# abs container name, required if you set resource.storage.type=ABS
+remote.logging.abs.container.name=<your-container-name>
 ```
 
 ### 注意事项

dolphinscheduler-api/pom.xml

@@ -61,6 +61,11 @@
             <artifactId>dolphinscheduler-meter</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-data-quality</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.dolphinscheduler</groupId>
             <artifactId>dolphinscheduler-datasource-all</artifactId>

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java

@@ -37,8 +37,7 @@ public class ApiFuncIdentificationConstant {
     public static final String TENANT_CREATE = "security:tenant:create";
     public static final String TENANT_UPDATE = "security:tenant:update";
     public static final String TENANT_DELETE = "security:tenant:delete";
-    public static final String ALART_LIST = "monitor:alert:view";
-    public static final String ALART_INSTANCE_CREATE = "security:alert-plugin:create";
+    public static final String ALERT_INSTANCE_CREATE = "security:alert-plugin:create";
     public static final String ALERT_PLUGIN_UPDATE = "security:alert-plugin:update";
     public static final String ALERT_PLUGIN_DELETE = "security:alert-plugin:delete";
     public static final String WORKER_GROUP_CREATE = "security:worker-group:create";

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java

@@ -162,7 +162,7 @@ public Result<Object> queryDataSource(@Parameter(hidden = true) @RequestAttribut
     @ApiException(QUERY_DATASOURCE_ERROR)
     public Result<Object> queryDataSourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
                                               @RequestParam("type") DbType type) {
-        List<DataSource> datasourceList = dataSourceService.queryDataSourceList(loginUser, type.ordinal());
+        List<DataSource> datasourceList = dataSourceService.queryDataSourceList(loginUser, type.getCode());
         return Result.success(datasourceList);
     }
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

@@ -554,7 +554,7 @@ public Result<Object> queryUdfFuncListPaging(@Parameter(hidden = true) @RequestA
     @ApiException(QUERY_DATASOURCE_BY_TYPE_ERROR)
     public Result<Object> queryUdfFuncList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
                                            @RequestParam("type") UdfType type) {
-        return udfFuncService.queryUdfFuncList(loginUser, type.ordinal());
+        return udfFuncService.queryUdfFuncList(loginUser, type.getCode());
     }
 
     /**

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@@ -323,6 +323,8 @@ public enum Status {
 
     REMOVE_TASK_INSTANCE_CACHE_ERROR(20019, "remove task instance cache error", "删除任务实例缓存错误"),
 
+    ILLEGAL_RESOURCE_PATH(20020, "Resource file [{0}] is illegal", "非法的资源路径[{0}]"),
+
     USER_NO_OPERATION_PERM(30001, "user has no operation privilege", "当前用户没有操作权限"),
     USER_NO_OPERATION_PROJECT_PERM(30002, "user {0} is not has project {1} permission", "当前用户[{0}]没有[{1}]项目的操作权限"),
     USER_NO_WRITE_PROJECT_PERM(30003, "user [{0}] does not have write permission for project [{1}]",

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@@ -194,13 +194,13 @@ Result<Object> updateResourceContent(User loginUser, String fullName, String ten
     org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException;
 
     /**
-     * Get resource by given resource type and full name.
+     * Get resource by given resource type and file name.
      * Useful in Python API create task which need processDefinition information.
      *
      * @param userName user who query resource
-     * @param fullName full name of the resource
+     * @param fileName file name of the resource
      */
-    StorageEntity queryFileStatus(String userName, String fullName) throws Exception;
+    StorageEntity queryFileStatus(String userName, String fileName) throws Exception;
 
     /**
      * delete DATA_TRANSFER data in resource center

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java

@@ -17,7 +17,7 @@
 
 package org.apache.dolphinscheduler.api.service.impl;
 
-import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALART_INSTANCE_CREATE;
+import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_INSTANCE_CREATE;
 import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_DELETE;
 import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_UPDATE;
 
@@ -108,7 +108,7 @@ public AlertPluginInstance create(User loginUser,
                                       WarningType warningType,
                                       String pluginInstanceParams) {
 
-        if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALART_INSTANCE_CREATE)) {
+        if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALERT_INSTANCE_CREATE)) {
             throw new ServiceException(Status.USER_NO_OPERATION_PERM);
         }
 
@@ -359,7 +359,7 @@ public void testSend(int pluginDefineId, String pluginInstanceParams) {
             throw new ServiceException(Status.ALERT_TEST_SENDING_FAILED, e.getMessage());
         }
 
-        if (alertSendResponse.isSuccess()) {
+        if (!alertSendResponse.isSuccess()) {
             throw new ServiceException(Status.ALERT_TEST_SENDING_FAILED,
                     alertSendResponse.getResResults().get(0).getMessage());
         }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java

@@ -230,7 +230,7 @@ public BaseDataSourceParamDTO queryDataSource(int id, User loginUser) {
     @Override
     public PageInfo<DataSource> queryDataSourceListPaging(User loginUser, String searchVal, Integer pageNo,
                                                           Integer pageSize) {
-        IPage<DataSource> dataSourceList = null;
+        IPage<DataSource> dataSourceList;
         Page<DataSource> dataSourcePage = new Page<>(pageNo, pageSize);
         PageInfo<DataSource> pageInfo = new PageInfo<>(pageNo, pageSize);
         if (loginUser.getUserType().equals(UserType.ADMIN_USER)) {
@@ -282,7 +282,7 @@ private String getHiddenPassword() {
     @Override
     public List<DataSource> queryDataSourceList(User loginUser, Integer type) {
 
-        List<DataSource> datasourceList = null;
+        List<DataSource> datasourceList;
         if (loginUser.getUserType().equals(UserType.ADMIN_USER)) {
             datasourceList = dataSourceMapper.queryDataSourceByType(0, type);
         } else {
@@ -420,7 +420,7 @@ public List<DataSource> authedDatasource(User loginUser, Integer userId) {
     public List<ParamsOptions> getTables(Integer datasourceId, String database) {
         DataSource dataSource = dataSourceMapper.selectById(datasourceId);
 
-        List<String> tableList = null;
+        List<String> tableList;
         BaseConnectionParam connectionParam =
                 (BaseConnectionParam) DataSourceUtils.buildConnectionParams(
                         dataSource.getType(),

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java

@@ -155,6 +155,8 @@ public Result updateProjectParameter(User loginUser, long projectCode, long code
 
         projectParameter.setParamName(projectParameterName);
         projectParameter.setParamValue(projectParameterValue);
+        projectParameter.setUpdateTime(new Date());
+        projectParameter.setOperator(loginUser.getId());
 
         if (projectParameterMapper.updateById(projectParameter) > 0) {
             log.info("Project parameter is updated and id is :{}", projectParameter.getId());

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

@@ -126,6 +126,7 @@ public Result<Object> createDirectory(User loginUser, String name, ResourceType
         }
 
         String tenantCode = getTenantCode(user);
+        checkFullName(tenantCode, currentDir);
 
         String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode)
                 : storageOperate.getResDir(tenantCode);
@@ -171,6 +172,7 @@ public Result<Object> uploadResource(User loginUser, String name, ResourceType t
         }
 
         String tenantCode = getTenantCode(user);
+        checkFullName(tenantCode, currentDir);
 
         result = verifyFile(name, type, file);
         if (!result.getCode().equals(Status.SUCCESS.getCode())) {
@@ -257,14 +259,15 @@ public Result<Object> updateResource(User loginUser, String resourceFullName, St
         }
 
         String tenantCode = getTenantCode(user);
+        checkFullName(tenantCode, resourceFullName);
 
         if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
             log.error("current user does not have permission");
             putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
             return result;
         }
 
-        String defaultPath = storageOperate.getResDir(tenantCode);
+        String defaultPath = storageOperate.getDir(type, tenantCode);
 
         StorageEntity resource;
         try {
@@ -949,6 +952,7 @@ public Result<Object> createResourceFile(User loginUser, ResourceType type, Stri
         }
 
         String tenantCode = getTenantCode(user);
+        checkFullName(tenantCode, currentDir);
 
         if (FileUtils.directoryTraversal(fileName)) {
             log.warn("File name verify failed, fileName:{}.", RegexUtils.escapeNRT(fileName));
@@ -1280,9 +1284,19 @@ private String getTenantCode(User user) {
     }
 
     private void checkFullName(String userTenantCode, String fullName) {
+        if (StringUtils.isEmpty(fullName)) {
+            return;
+        }
+        if (FOLDER_SEPARATOR.equalsIgnoreCase(fullName)) {
+            return;
+        }
+        // Avoid returning to the parent directory
+        if (fullName.contains("../")) {
+            throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName);
+        }
         String baseDir = storageOperate.getDir(ResourceType.ALL, userTenantCode);
-        if (StringUtils.isNotBlank(fullName) && !StringUtils.startsWith(fullName, baseDir)) {
-            throw new ServiceException("Resource file: " + fullName + " is illegal");
+        if (!StringUtils.startsWith(fullName, baseDir)) {
+            throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName);
         }
     }
 }