Unverified Commit 72408b26 authored by BoYiZhang's avatar BoYiZhang Committed by GitHub
Browse files

Merge pull request #9 from apache/dev 

update code
parents b9f7da80 1a78c115

.github/workflows/ci_e2e.yml

+6 −7
Original line number Diff line number Diff line
@@ -44,15 +44,14 @@ jobs: 
            ${{ runner.os }}-maven-

      - name: Build Image

        run: |

          export VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`

          sh ./dockerfile/hooks/build

          sh ./docker/build/hooks/build

      - name: Docker Run

        run: |

          VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`

          mkdir -p /tmp/logs

          docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all

          export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')

          sed -i "s/apache\/dolphinscheduler:latest/apache\/dolphinscheduler:${VERSION}/g" $(pwd)/docker/docker-swarm/docker-compose.yml

          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d

      - name: Check Server Status

        run: sh ./dockerfile/hooks/check

        run: sh $(pwd)/docker/docker-swarm/check

      - name: Prepare e2e env

        run: |

          sudo apt-get install -y libxss1 libappindicator1 libindicator7 xvfb unzip libgbm1
@@ -70,6 +69,6 @@ jobs: 
        uses: actions/upload-artifact@v1

        with:

          name: dslogs

          path: /tmp/logs

          path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data

.github/workflows/ci_ut.yml

+6 −4
Original line number Diff line number Diff line
@@ -21,7 +21,6 @@ on: 
    branches:

      - dev

env:

  DOCKER_DIR: ./docker

  LOG_DIR: /tmp/dolphinscheduler



name: Unit Test
@@ -47,7 +46,11 @@ jobs: 
          restore-keys: |

            ${{ runner.os }}-maven-

      - name: Bootstrap database

        run: cd ${DOCKER_DIR} && docker-compose up -d

        run: |

          sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml

          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml create --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql

          sudo cp $(pwd)/sql/dolphinscheduler-postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}')

          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql

      - name: Set up JDK 1.8

        uses: actions/setup-java@v1

        with:
@@ -82,6 +85,5 @@ jobs: 
      - name: Collect logs

        run: |

          mkdir -p ${LOG_DIR}

          cd ${DOCKER_DIR}

          docker-compose logs db > ${LOG_DIR}/db.txt

          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt

        continue-on-error: true

dockerfile/Dockerfile→docker/build/Dockerfile

+11 −25
Original line number Diff line number Diff line
@@ -37,32 +37,19 @@ RUN apk add openjdk8 
ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk

ENV PATH $JAVA_HOME/bin:$PATH



#3. install zk

RUN cd /opt && \

    wget https://downloads.apache.org/zookeeper/zookeeper-3.5.7/apache-zookeeper-3.5.7-bin.tar.gz && \

    tar -zxvf apache-zookeeper-3.5.7-bin.tar.gz && \

    mv apache-zookeeper-3.5.7-bin zookeeper && \

    mkdir -p /tmp/zookeeper && \

    rm -rf ./zookeeper-*tar.gz && \

    rm -rf /opt/zookeeper/conf/zoo_sample.cfg

ADD ./conf/zookeeper/zoo.cfg /opt/zookeeper/conf

ENV ZK_HOME /opt/zookeeper

ENV PATH $ZK_HOME/bin:$PATH

#3. add dolphinscheduler

ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/

RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/

ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler



#4. install pg

RUN apk add postgresql postgresql-contrib



#5. add dolphinscheduler

ADD ./apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz /opt/

RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin/ /opt/dolphinscheduler/

ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler



#6. modify nginx

#5. modify nginx

RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \

    rm -rf /etc/nginx/conf.d/*

ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d



#7. add configuration and modify permissions and set soft links

#6. add configuration and modify permissions and set soft links

ADD ./checkpoint.sh /root/checkpoint.sh

ADD ./startup-init-conf.sh /root/startup-init-conf.sh

ADD ./startup.sh /root/startup.sh
@@ -75,22 +62,21 @@ RUN chmod +x /root/checkpoint.sh && \ 
    chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \

    chmod +x /opt/dolphinscheduler/script/*.sh && \

    chmod +x /opt/dolphinscheduler/bin/*.sh && \

    chmod +x /opt/zookeeper/bin/*.sh && \

    dos2unix /root/checkpoint.sh && \

    dos2unix /root/startup-init-conf.sh && \

    dos2unix /root/startup.sh && \

    dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \

    dos2unix /opt/dolphinscheduler/script/*.sh && \

    dos2unix /opt/dolphinscheduler/bin/*.sh && \

    dos2unix /opt/zookeeper/bin/*.sh && \

    rm -rf /bin/sh && \

    ln -s /bin/bash /bin/sh && \

    mkdir -p /tmp/xls



#8. remove apk index cache

RUN rm -rf /var/cache/apk/*

#7. remove apk index cache and disable coredup for sudo

RUN rm -rf /var/cache/apk/* && \

    echo "Set disable_coredump false" >> /etc/sudo.conf



#9. expose port

#8. expose port

EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888



ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]

dockerfile/README.md→docker/build/README.md

+3 −3
Original line number Diff line number Diff line
@@ -109,16 +109,16 @@ In Unix-Like, Example: 


```bash

$ cd path/incubator-dolphinscheduler

$ sh ./dockerfile/hooks/build

$ sh ./docker/build/hooks/build

```



In Windows, Example:



```bat

c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat

c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat

```



Please read `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat` script files if you don't understand

Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand



## Environment Variables

dockerfile/README_zh_CN.md→docker/build/README_zh_CN.md

+3 −3
Original line number Diff line number Diff line
@@ -109,16 +109,16 @@ dolphinscheduler frontend 


```bash

$ cd path/incubator-dolphinscheduler

$ sh ./dockerfile/hooks/build

$ sh ./docker/build/hooks/build

```



Windows系统, 如下:



```bat

c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat

c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat

```



如果你不理解这些脚本 `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat`，请阅读里面的内容。

如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`，请阅读里面的内容。



## 环境变量