Unverified Commit 98fdba67 authored by LEI SHENG's avatar LEI SHENG Committed by GitHub
Browse files

[Bug 2923] Hive JDBC connection parameter ignored (#3194) 



* fix the bug #2923 (Hive JDBC connection parameter ignored) , the hive jdbc connection url ;

* fix the bug index out of bound and add some test case

* add the Licensed

* fix the checkstyle and import StringBuilder

* put HiveDataSourceTest.java to root pom maven-surefire-plugin to avoiding the sonar 0.0% Coverage.

* Update HiveDataSource.java

Co-authored-by: default avatardailidong <dailidong66@gmail.com>
parent 6d43c21d

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java

+53 −0
Original line number Diff line number Diff line
@@ -14,10 +14,16 @@ 
 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package org.apache.dolphinscheduler.dao.datasource;



import org.apache.dolphinscheduler.common.Constants;

import org.apache.dolphinscheduler.common.enums.DbType;

import org.apache.dolphinscheduler.common.utils.StringUtils;

import java.util.Set;

import java.util.stream.Collectors;

import java.util.stream.Stream;

import org.apache.hadoop.hive.conf.HiveConf.ConfVars;



/**

 * data source of hive
@@ -40,4 +46,51 @@ public class HiveDataSource extends BaseDataSource { 
  public DbType dbTypeSelector() {

    return DbType.HIVE;

  }



  /**

   * build hive jdbc params,append : ?hive_conf_list

   *

   * hive jdbc url template:

   *

   * jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list

   *

   * @param otherParams otherParams

   * @return filter otherParams

   */

  @Override

  protected String filterOther(String otherParams) {

    if (StringUtils.isBlank(otherParams)) {

      return "";

    }



    StringBuilder hiveConfListSb = new StringBuilder();

    hiveConfListSb.append("?");

    StringBuilder sessionVarListSb = new StringBuilder();



    String[] otherArray = otherParams.split(";", -1);



    // get the default hive conf var name

    Set<String> hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname)

        .collect(Collectors.toSet());



    for (String conf : otherArray) {

      if (hiveConfSet.contains(conf.split("=")[0])) {

        hiveConfListSb.append(conf).append(";");

      } else {

        sessionVarListSb.append(conf).append(";");

      }

    }



    // remove the last ";"

    if (sessionVarListSb.length() > 0) {

      sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1);

    }



    if (hiveConfListSb.length() > 0) {

      hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1);

    }



    return sessionVarListSb.toString() + hiveConfListSb.toString();

  }

  

}

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java

0 → 100644
+85 −0
Original line number Diff line number Diff line 
/*

 * Licensed to the Apache Software Foundation (ASF) under one or more

 * contributor license agreements.  See the NOTICE file distributed with

 * this work for additional information regarding copyright ownership.

 * The ASF licenses this file to You under the Apache License, Version 2.0

 * (the "License"); you may not use this file except in compliance with

 * the License.  You may obtain a copy of the License at

 *

 *    http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package org.apache.dolphinscheduler.dao.datasource;



import org.junit.Assert;

import org.junit.Test;



/**

 * test data source of hive

 */

public class HiveDataSourceTest {



  @Test

  public void testfilterOther() {

    BaseDataSource hiveDataSource = new HiveDataSource();



    // not contain hive_site_conf

    String other = hiveDataSource.filterOther("charset=UTF-8");

    Assert.assertEquals("charset=UTF-8", other);



    // not contain

    other = hiveDataSource.filterOther("");

    Assert.assertEquals("", other);



    // only contain hive_site_conf

    other = hiveDataSource.filterOther("hive.mapred.mode=strict");

    Assert.assertEquals("?hive.mapred.mode=strict", other);



    // contain hive_site_conf at the first

    other = hiveDataSource.filterOther("hive.mapred.mode=strict;charset=UTF-8");

    Assert.assertEquals("charset=UTF-8?hive.mapred.mode=strict", other);



    // contain hive_site_conf in the middle

    other = hiveDataSource.filterOther("charset=UTF-8;hive.mapred.mode=strict;foo=bar");

    Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other);



    // contain hive_site_conf at the end

    other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict");

    Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other);



    // contain multi hive_site_conf

    other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict;hive.exec.parallel=true");

    Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict;hive.exec.parallel=true", other);

  }



  @Test

  public void testGetHiveJdbcUrlOther() {



    BaseDataSource hiveDataSource = new HiveDataSource();

    hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000");

    hiveDataSource.setDatabase("test");

    hiveDataSource.setPassword("123456");

    hiveDataSource.setUser("test");

    Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl());



    hiveDataSource.setOther("charset=UTF-8;hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");



    Assert.assertEquals(

        "jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",

        hiveDataSource.getJdbcUrl());



    hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");



    Assert.assertEquals(

        "jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",

        hiveDataSource.getJdbcUrl());



  }



}
 
 No newline at end of file

pom.xml

+1 −0
Original line number Diff line number Diff line
@@ -852,6 +852,7 @@ 
                        <include>**/dao/utils/DagHelperTest.java</include>

                        <include>**/dao/AlertDaoTest.java</include>

                        <include>**/dao/datasource/OracleDataSourceTest.java</include>

                        <include>**/dao/datasource/HiveDataSourceTest.java</include>

                        <include>**/dao/upgrade/ProcessDefinitionDaoTest.java</include>

                        <include>**/dao/upgrade/WokrerGrouopDaoTest.java</include>

                        <include>**/dao/upgrade/UpgradeDaoTest.java</include>