Unverified Commit c1ee1333 authored by qiaozhanwei's avatar qiaozhanwei Committed by GitHub
Browse files

Refactor Architecture Basic modification #1658 (#1946) 



* 1，remove dolphinscheduler-rpc module  2，add dolphinscheduler-remote module 3，add dolphinscheduler-service module 4，refactor LoggerServer module (#1925)

* 1，remove dolphinscheduler-rpc module
2，add dolphinscheduler-remote module
3，add dolphinscheduler-service module
4，refactor LoggerServer module

* ProcessUtils modify

* Refactor architecture (#1926)

* move version to parent pom

* move version properties to parent pom for easy management

* remove freemarker dependency

* delete CombinedApplicationServer

* #1871 correct spelling

* #1873 some updates for TaskQueueZkImpl

* #1875 remove unused properties in pom

* #1878
1. remove tomcat dependency
2. remove combined_logback.xml in api module
3. format pom.xml for not aligning

* #1885 fix api server startup failure
1. add jsp-2.1 dependency
2. remove jasper-runtime dependency

* add stringutils ut (#1921)

* add stringutils ut

* Newfeature for #1675. (#1908)

Continue to finish the rest works, add the cache feature for dependence,mr,python,sub_process,procedure and shell.

* Add modify user name for process definition (#1919)

* class overrides equals() and should therefore also override hashCode()

* #1862 add modify user in process difinition list

* #1862 add pg-1.2.2 ddl.sql

* modify ScriptRunnerTest

* add updateProessDifinition UT

* modify updateProcessDifinition UT

* modify updateProcessDifinition UT

* modify mysql 1.2.2 ddl.sql&dml.sql

* add scope test to mysql in pom

* modify pg-1.2.2 ddl.sql

* refactor module

* updates

Co-authored-by: default avatarkhadgarmage <khadgar.mage@outlook.com>
Co-authored-by: default avatarzhukai <boness@qq.com>
Co-authored-by: default avatarYelli <amarantine@my.com>

* dolphinscheduler-common remove spring (#1931)

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* SpringApplicationContext class title add license (#1932)

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* add license (#1934)

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* Refactor architecture (#1936)

* move datasource classes to dao module

* fix send4LetterWord bug

* LoggerServiceTest remove ProcessDao (#1944)

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* dolphinscheduler-common remove spring

* LoggerServiceTest remove ProcessDao

* exclude jasper-compiler in case of runtime conflict (#1938)

* move datasource classes to dao module

* fix send4LetterWord bug

* exclude jasper-compiler in case of runtime conflict

* DataAnaylysisServiceTest and ProcessDefinitionService modify

* remote module add comment

* OSUtilsTest modify

* add finally block to close channel (#1951)

* move datasource classes to dao module

* fix send4LetterWord bug

* exclude jasper-compiler in case of runtime conflict

* add finally block to close channel

Co-authored-by: default avatarTboy <technoboy@yeah.net>
Co-authored-by: default avatarkhadgarmage <khadgar.mage@outlook.com>
Co-authored-by: default avatarzhukai <boness@qq.com>
Co-authored-by: default avatarYelli <amarantine@my.com>
parent cdd53a76

dolphinscheduler-api/pom.xml

+4 −10
Original line number Diff line number Diff line
@@ -31,12 +31,6 @@ 
    <dependency>

      <groupId>org.apache.dolphinscheduler</groupId>

      <artifactId>dolphinscheduler-alert</artifactId>

      <exclusions>

        <exclusion>

          <groupId>org.apache.dolphinscheduler</groupId>

          <artifactId>dolphinscheduler-dao</artifactId>

        </exclusion>

      </exclusions>

    </dependency>



    <dependency>
@@ -129,13 +123,13 @@ 
    </dependency>



    <dependency>

      <groupId>com.github.xiaoymin</groupId>

      <artifactId>swagger-bootstrap-ui</artifactId>

      <groupId>org.apache.dolphinscheduler</groupId>

      <artifactId>dolphinscheduler-service</artifactId>

    </dependency>



    <dependency>

      <groupId>org.apache.dolphinscheduler</groupId>

      <artifactId>dolphinscheduler-rpc</artifactId>

      <groupId>com.github.xiaoymin</groupId>

      <artifactId>swagger-bootstrap-ui</artifactId>

    </dependency>



    <dependency>

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java

+2 −2
Original line number Diff line number Diff line
@@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.api.utils.Result; 
import org.apache.dolphinscheduler.common.Constants;

import org.apache.dolphinscheduler.common.enums.ExecutionStatus;

import org.apache.dolphinscheduler.common.enums.Flag;

import org.apache.dolphinscheduler.common.queue.ITaskQueue;

import org.apache.dolphinscheduler.common.queue.TaskQueueFactory;

import org.apache.dolphinscheduler.common.utils.ParameterUtils;

import org.apache.dolphinscheduler.common.utils.StringUtils;

import org.apache.dolphinscheduler.dao.entity.User;

import io.swagger.annotations.*;

import org.apache.dolphinscheduler.service.queue.ITaskQueue;

import org.apache.dolphinscheduler.service.queue.TaskQueueFactory;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import org.springframework.beans.factory.annotation.Autowired;

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/log/LogClient.java

deleted100644 → 0
+0 −137
Original line number Diff line number Diff line 
/*

 * Licensed to the Apache Software Foundation (ASF) under one or more

 * contributor license agreements.  See the NOTICE file distributed with

 * this work for additional information regarding copyright ownership.

 * The ASF licenses this file to You under the Apache License, Version 2.0

 * (the "License"); you may not use this file except in compliance with

 * the License.  You may obtain a copy of the License at

 *

 *    http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package org.apache.dolphinscheduler.api.log;



import io.grpc.ManagedChannel;

import io.grpc.ManagedChannelBuilder;

import io.grpc.StatusRuntimeException;

import org.apache.dolphinscheduler.rpc.*;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import java.util.concurrent.TimeUnit;



/**

 * log client

 */

public class LogClient {



    private static final Logger logger = LoggerFactory.getLogger(LogClient.class);



    private final ManagedChannel channel;

    private final LogViewServiceGrpc.LogViewServiceBlockingStub blockingStub;



    /**

     * construct client connecting to HelloWorld server at {@code host:port}

     *

     * @param host host

     * @param port port

     */

    public LogClient(String host, int port) {

        this(ManagedChannelBuilder.forAddress(host, port)

                // Channels are secure by default (via SSL/TLS). For the example we disable TLS to avoid

                // needing certificates.

                .usePlaintext(true));

    }



    /**

     * construct client for accessing RouteGuide server using the existing channel

     *

     */

    LogClient(ManagedChannelBuilder<?> channelBuilder) {

        /**

         * set max read size

         */

        channelBuilder.maxInboundMessageSize(Integer.MAX_VALUE);

        channel = channelBuilder.build();

        blockingStub = LogViewServiceGrpc.newBlockingStub(channel);

    }



    /**

     * shutdown

     *

     * @throws InterruptedException InterruptedException

     */

    public void shutdown() throws InterruptedException {

        channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);

    }



    /**

     * roll view log

     *

     * @param path path

     * @param skipLineNum skip line number

     * @param limit limit

     * @return log content

     */

    public String rollViewLog(String path,int skipLineNum,int limit) {

        logger.info("roll view log : path {},skipLineNum {} ,limit {}", path, skipLineNum, limit);

        LogParameter pathParameter = LogParameter

                .newBuilder()

                .setPath(path)

                .setSkipLineNum(skipLineNum)

                .setLimit(limit)

                .build();

        RetStrInfo retStrInfo;

        try {

            retStrInfo = blockingStub.rollViewLog(pathParameter);

            return retStrInfo.getMsg();

        } catch (StatusRuntimeException e) {

            logger.error("roll view log error", e);

            return null;

        }

    }



    /**

     * view log

     *

     * @param path path

     * @return log content

     */

    public String viewLog(String path) {

        logger.info("view log path {}",path);

        PathParameter pathParameter = PathParameter.newBuilder().setPath(path).build();

        RetStrInfo retStrInfo;

        try {

            retStrInfo = blockingStub.viewLog(pathParameter);

            return retStrInfo.getMsg();

        } catch (StatusRuntimeException e) {

            logger.error("view log error", e);

            return null;

        }

    }



    /**

     * get log size

     *

     * @param path log path

     * @return log content bytes

     */

    public byte[] getLogBytes(String path) {

        logger.info("log path {}",path);

        PathParameter pathParameter = PathParameter.newBuilder().setPath(path).build();

        RetByteInfo retByteInfo;

        try {

            retByteInfo = blockingStub.getLogBytes(pathParameter);

            return retByteInfo.getData().toByteArray();

        } catch (StatusRuntimeException e) {

            logger.error("log size error", e);

            return null;

        }

    }



}
 
 No newline at end of file

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java

+5 −5
Original line number Diff line number Diff line
@@ -24,13 +24,13 @@ import org.apache.dolphinscheduler.api.enums.Status; 
import org.apache.dolphinscheduler.common.Constants;

import org.apache.dolphinscheduler.common.enums.CommandType;

import org.apache.dolphinscheduler.common.enums.UserType;

import org.apache.dolphinscheduler.common.queue.ITaskQueue;

import org.apache.dolphinscheduler.common.queue.TaskQueueFactory;

import org.apache.dolphinscheduler.common.utils.DateUtils;

import org.apache.dolphinscheduler.common.utils.StringUtils;

import org.apache.dolphinscheduler.dao.ProcessDao;

import org.apache.dolphinscheduler.dao.entity.*;

import org.apache.dolphinscheduler.dao.mapper.*;

import org.apache.dolphinscheduler.service.process.ProcessService;

import org.apache.dolphinscheduler.service.queue.ITaskQueue;

import org.apache.dolphinscheduler.service.queue.TaskQueueFactory;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import org.springframework.beans.factory.annotation.Autowired;
@@ -69,7 +69,7 @@ public class DataAnalysisService extends BaseService{ 
    TaskInstanceMapper taskInstanceMapper;



    @Autowired

    ProcessDao processDao;

    ProcessService processService;



    /**

     * statistical task instance status data
@@ -296,7 +296,7 @@ public class DataAnalysisService extends BaseService{ 
        if(projectId !=0){

            projectIds.add(projectId);

        }else if(loginUser.getUserType() == UserType.GENERAL_USER){

            projectIds = processDao.getProjectIdListHavePerm(loginUser.getId());

            projectIds = processService.getProjectIdListHavePerm(loginUser.getId());

            if(projectIds.size() ==0 ){

                projectIds.add(0);

            }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

+1 −2
Original line number Diff line number Diff line
@@ -21,10 +21,9 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; 
import org.apache.dolphinscheduler.api.utils.Result;

import org.apache.dolphinscheduler.common.Constants;

import org.apache.dolphinscheduler.common.enums.DbType;

import org.apache.dolphinscheduler.common.enums.UserType;

import org.apache.dolphinscheduler.common.job.db.*;

import org.apache.dolphinscheduler.common.utils.CommonUtils;

import org.apache.dolphinscheduler.common.utils.JSONUtils;

import org.apache.dolphinscheduler.dao.datasource.*;

import org.apache.dolphinscheduler.dao.entity.DataSource;

import org.apache.dolphinscheduler.dao.entity.Resource;

import org.apache.dolphinscheduler.dao.entity.User;