Unverified Commit f56ae9e6 authored by BoYiZhang's avatar BoYiZhang Committed by GitHub
Browse files

add http task timeout & test unit (#2955) 



* add http task timeout & test unit

* add http task timeout & test unit

Co-authored-by: default avatarzhanglong <zhanglong@ysstech.com>
Co-authored-by: default avatardailidong <dailidong66@gmail.com>
parent 284c50f6

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java

+27 −0
Original line number Diff line number Diff line
@@ -56,6 +56,17 @@ public class HttpParameters extends AbstractParameters { 
    private String condition;





    /**

     * Connect Timeout

     * Unit: ms

     */

    private int connectTimeout ;



    /**

     * Socket Timeout

     * Unit: ms

     */

    private int socketTimeout ;



    @Override

    public boolean checkParameters() {
@@ -106,4 +117,20 @@ public class HttpParameters extends AbstractParameters { 
    public void setCondition(String condition) {

        this.condition = condition;

    }



    public int getConnectTimeout() {

        return connectTimeout;

    }



    public void setConnectTimeout(int connectTimeout) {

        this.connectTimeout = connectTimeout;

    }



    public int getSocketTimeout() {

        return socketTimeout;

    }



    public void setSocketTimeout(int socketTimeout) {

        this.socketTimeout = socketTimeout;

    }

}

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/HttpParametersTest.java

0 → 100644
+91 −0
Original line number Diff line number Diff line 
/*

 * Licensed to the Apache Software Foundation (ASF) under one or more

 * contributor license agreements.  See the NOTICE file distributed with

 * this work for additional information regarding copyright ownership.

 * The ASF licenses this file to You under the Apache License, Version 2.0

 * (the "License"); you may not use this file except in compliance with

 * the License.  You may obtain a copy of the License at

 *

 *    http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package org.apache.dolphinscheduler.common.task;



import java.util.ArrayList;

import java.util.List;



import org.apache.commons.lang.StringUtils;

import org.apache.dolphinscheduler.common.enums.HttpCheckCondition;

import org.apache.dolphinscheduler.common.enums.HttpMethod;

import org.apache.dolphinscheduler.common.process.HttpProperty;

import org.apache.dolphinscheduler.common.process.ResourceInfo;

import org.apache.dolphinscheduler.common.task.http.HttpParameters;

import org.junit.Assert;

import org.junit.Test;



import com.alibaba.fastjson.JSON;



/**

 * http parameter

 */

public class HttpParametersTest  {





    @Test

    public void testGenerator(){

        String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," +

                "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";

        HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class);





        Assert.assertEquals(10000,httpParameters.getConnectTimeout() );

        Assert.assertEquals(10000,httpParameters.getSocketTimeout());

        Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl());

        Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());

        Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());

        Assert.assertEquals("",httpParameters.getCondition());



    }





    @Test

    public void testCheckParameters(){

        String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," +

                "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";

        HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class);



        Assert.assertTrue( httpParameters.checkParameters());

        Assert.assertEquals(10000,httpParameters.getConnectTimeout() );

        Assert.assertEquals(10000,httpParameters.getSocketTimeout());

        Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl());

        Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());

        Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());

        Assert.assertEquals("",httpParameters.getCondition());



    }





    @Test

    public void testCheckValues() {

        String paramData = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://www.baidu.com/\"," +

                "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";

        HttpParameters httpParameters = JSON.parseObject(paramData, HttpParameters.class);



        Assert.assertTrue( httpParameters.checkParameters());

        Assert.assertEquals(10000,httpParameters.getConnectTimeout() );

        Assert.assertEquals(10000,httpParameters.getSocketTimeout());

        Assert.assertEquals("https://www.baidu.com/",httpParameters.getUrl());

        Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());

        Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());

        Assert.assertEquals("",httpParameters.getCondition());

        Assert.assertEquals(0,httpParameters.getLocalParametersMap().size());

        Assert.assertEquals(0,httpParameters.getResourceFilesList().size());

    }





}

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java

+1 −8
Original line number Diff line number Diff line
@@ -66,13 +66,6 @@ public class HttpTask extends AbstractTask { 
     */

    private HttpParameters httpParameters;







    /**

     * Convert mill seconds to second unit

     */

    protected static final int MAX_CONNECTION_MILLISECONDS = 60000;



    /**

     * application json

     */
@@ -303,7 +296,7 @@ public class HttpTask extends AbstractTask { 
     * @return RequestConfig

     */

    private RequestConfig requestConfig() {

        return RequestConfig.custom().setSocketTimeout(MAX_CONNECTION_MILLISECONDS).setConnectTimeout(MAX_CONNECTION_MILLISECONDS).build();

        return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()).setConnectTimeout(httpParameters.getConnectTimeout()).build();

    }



    /**

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java

0 → 100644
+208 −0
Original line number Diff line number Diff line 
/*

 * Licensed to the Apache Software Foundation (ASF) under one or more

 * contributor license agreements.  See the NOTICE file distributed with

 * this work for additional information regarding copyright ownership.

 * The ASF licenses this file to You under the Apache License, Version 2.0

 * (the "License"); you may not use this file except in compliance with

 * the License.  You may obtain a copy of the License at

 *

 *    http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package org.apache.dolphinscheduler.server.worker.task.http;



import static org.apache.dolphinscheduler.common.enums.CommandType.*;



import java.io.IOException;

import java.util.Date;



import org.apache.dolphinscheduler.common.enums.HttpCheckCondition;

import org.apache.dolphinscheduler.common.enums.HttpMethod;

import org.apache.dolphinscheduler.common.task.http.HttpParameters;

import org.apache.dolphinscheduler.common.utils.OSUtils;

import org.apache.dolphinscheduler.dao.entity.ProcessInstance;

import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;

import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;

import org.apache.dolphinscheduler.server.worker.task.TaskProps;

import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;

import org.apache.dolphinscheduler.service.process.ProcessService;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.RequestBuilder;

import org.apache.http.impl.client.CloseableHttpClient;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mockito;

import org.powermock.api.mockito.PowerMockito;

import org.powermock.core.classloader.annotations.PowerMockIgnore;

import org.powermock.core.classloader.annotations.PrepareForTest;

import org.powermock.modules.junit4.PowerMockRunner;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import org.springframework.context.ApplicationContext;



import com.alibaba.fastjson.JSON;



@RunWith(PowerMockRunner.class)

@PrepareForTest(OSUtils.class)

@PowerMockIgnore({"javax.management.*","javax.net.ssl.*"})

public class HttpTaskTest {

    private static final Logger logger = LoggerFactory.getLogger(HttpTaskTest.class);







    private HttpTask httpTask;



    private ProcessService processService;



    private ShellCommandExecutor shellCommandExecutor;



    private ApplicationContext applicationContext;

    private TaskExecutionContext taskExecutionContext;



    @Before

    public void before() throws Exception {

        taskExecutionContext = new TaskExecutionContext();



        PowerMockito.mockStatic(OSUtils.class);

        processService = PowerMockito.mock(ProcessService.class);

        shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class);



        applicationContext = PowerMockito.mock(ApplicationContext.class);

        SpringApplicationContext springApplicationContext = new SpringApplicationContext();

        springApplicationContext.setApplicationContext(applicationContext);

        PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);



        TaskProps props = new TaskProps();

        props.setExecutePath("/tmp");

        props.setTaskAppId(String.valueOf(System.currentTimeMillis()));

        props.setTaskInstanceId(1);

        props.setTenantCode("1");

        props.setEnvFile(".dolphinscheduler_env.sh");

        props.setTaskStartTime(new Date());

        props.setTaskTimeout(0);

        props.setTaskParams(

                "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\",\"httpMethod\":\"GET\"," +

                        "\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"https://github.com/\"," +

                        "\"connectTimeout\":\"1000\",\"socketTimeout\":\"1000\"}");





        taskExecutionContext = Mockito.mock(TaskExecutionContext.class);

        Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams());

        Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp");

        Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("1");

        Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root");

        Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date());

        Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000);

        Mockito.when(taskExecutionContext.getLogPath()).thenReturn("/tmp/dx");



        httpTask = new HttpTask(taskExecutionContext, logger);

        httpTask.init();



    }



    @Test

    public void testGetParameters() {

        Assert.assertNotNull(httpTask.getParameters());

    }





    @Test

    public void testCheckParameters() {

        Assert.assertTrue(httpTask.getParameters().checkParameters());

    }





    @Test

    public void testGenerator(){

        String paramJson = "{\"localParams\":[],\"httpParams\":[],\"url\":\"https://github.com/\"," +

                "\"httpMethod\":\"GET\",\"httpCheckCondition\":\"STATUS_CODE_DEFAULT\",\"condition\":\"\",\"connectTimeout\":\"10000\",\"socketTimeout\":\"10000\"}";

        HttpParameters httpParameters = JSON.parseObject(paramJson, HttpParameters.class);





        Assert.assertEquals(10000,httpParameters.getConnectTimeout() );

        Assert.assertEquals(10000,httpParameters.getSocketTimeout());

        Assert.assertEquals("https://github.com/",httpParameters.getUrl());

        Assert.assertEquals(HttpMethod.GET,httpParameters.getHttpMethod());

        Assert.assertEquals(HttpCheckCondition.STATUS_CODE_DEFAULT,httpParameters.getHttpCheckCondition());

        Assert.assertEquals("",httpParameters.getCondition());



    }



    @Test

    public void testHandle(){

        boolean flag = true ;

        try {

            httpTask.handle();

        } catch (Exception e) {

            flag = false ;

            e.printStackTrace();

        }



        Assert.assertTrue(flag);



    }



    @Test

    public void testSendRequest(){



        CloseableHttpClient client = httpTask.createHttpClient();



        String statusCode = null;

        String body = null;



        try {



            CloseableHttpResponse response = httpTask.sendRequest(client) ;

            statusCode = String.valueOf(httpTask.getStatusCode(response));

            body = httpTask.getResponseBody(response);

            int exitStatusCode = httpTask.validResponse(body, statusCode);



            Assert.assertNotEquals(-1,exitStatusCode);



        } catch (IOException e) {

            e.printStackTrace();

        };

    }



    @Test

    public void testValidResponse(){

        String body = "body";

        String statusCode = "200" ;



        int exitStatusCode = httpTask.validResponse(body,statusCode);

        Assert.assertNotEquals(-1,exitStatusCode);



    }



    @Test

    public void testAppendMessage(){

        httpTask.appendMessage("message");



        Assert.assertEquals("message",httpTask.getOutput());

    }



    @Test

    public void testCreateHttpClient(){

        Assert.assertNotNull(httpTask.createHttpClient());

    }



    @Test

    public void testCreateRequestBuilder(){

        RequestBuilder  requestBuilder = httpTask.createRequestBuilder();

        Assert.assertEquals(RequestBuilder.get().getMethod(),requestBuilder.getMethod());

    }



    private ProcessInstance getProcessInstance() {

        ProcessInstance processInstance = new ProcessInstance();

        processInstance.setCommandType(START_PROCESS);

        processInstance.setScheduleTime(new Date());

        return processInstance;

    }

}

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/http.vue

+56 −2
Original line number Diff line number Diff line
@@ -85,6 +85,39 @@ 
        </x-input>

      </div>

    </m-list-box>





    <m-list-box  >

      <div slot="text">{{$t('Timeout Settings')}}</div>

      <div slot="content">

        <label class="label-box">

          <div style="padding-top: 5px;">

            <x-switch

              v-model="timeoutSettings"

              :disabled="isDetails"

            ></x-switch>

          </div>

        </label>

      </div>

    </m-list-box>



    <div class="clearfix list" v-if = "timeoutSettings" >

      <div class="text-box">

        <span>{{$t('Connect Timeout')}}</span>

      </div>

      <div class="cont-box">

        <span  class="label-box"  style="width: 193px;display: inline-block;" >

          <x-input v-model='connectTimeout' maxlength="7" />

        </span>

        <span>{{$t('ms')}}</span>

        <span class="text-b">{{$t('Socket Timeout')}}</span>

        <span  class="label-box" style="width: 193px;display: inline-block;" >

          <x-input v-model='socketTimeout' maxlength="7" />

        </span>

        <span>{{$t('ms')}}</span>

      </div>

    </div>



    <m-list-box>

      <div slot="text">{{$t('Custom Parameters')}}</div>

      <div slot="content">
@@ -110,6 +143,10 @@ 
    name: 'http',

    data () {

      return {

        timeoutSettings: false,

        connectTimeout : 60000 ,

        socketTimeout :  60000 ,



        url: '',

        condition: '',

        localParams: [],
@@ -152,6 +189,14 @@ 
        if (!this.$refs.refHttpParams._verifValue()) {

          return false

        }

        if (!_.isNumber(parseInt(this.socketTimeout))) {

          this.$message.warning(`${i18n.$t('Socket Timeout be a positive integer')}`)

          return false

        }

        if (!_.isNumber(parseInt(this.connectTimeout))) {

          this.$message.warning(`${i18n.$t('Connect timeout be a positive integer')}`)

          return false

        }

        // storage

        this.$emit('on-params', {

          localParams: this.localParams,
@@ -159,7 +204,9 @@ 
          url: this.url,

          httpMethod: this.httpMethod,

          httpCheckCondition: this.httpCheckCondition,

          condition: this.condition

          condition: this.condition,

          connectTimeout : this.connectTimeout ,

          socketTimeout : this.socketTimeout

        })

        return true

      }
@@ -172,7 +219,9 @@ 
          url: this.url,

          httpMethod: this.httpMethod,

          httpCheckCondition: this.httpCheckCondition,

          condition: this.condition

          condition: this.condition,

          connectTimeout : this.connectTimeout ,

          socketTimeout : this.socketTimeout

        }

      }

    },
@@ -193,6 +242,11 @@ 
          this.httpMethod = o.params.httpMethod || 'GET'

          this.httpCheckCondition = o.params.httpCheckCondition || 'DEFAULT'

          this.condition = o.params.condition || ''

          this.connectTimeout = o.params.connectTimeout

          this.socketTimeout = o.params.socketTimeout

          if(this.connectTimeout != 60000 || this.socketTimeout != 60000 ){

            this.timeoutSettings = true

          }

          // backfill localParams

          let localParams = o.params.localParams || []

          if (localParams.length) {