.gitlab-ci.yml Practice

Example:

stages:
  - prepare
  - build
  - training
  - test

variables:
  MASTER_DIR: "$CI_PROJECT_DIR/WORKING_DIR"
  IDF_PATH: "$CI_PROJECT_DIR/WORKING_DIR/esp-idf"
  BATCH_BUILD: "1"
  V: "0"

before_script:
  # add gitlab ssh key
  - mkdir -p ~/.ssh
  - chmod 700 ~/.ssh
  - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
  - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
  - chmod 600 ~/.ssh/id_rsa
  - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config

.do_nothing_before:
  before_script: &do_nothing_before
    - echo "do nothing"

preparation:
  stage: prepare
  tags:
    - build
  artifacts:
    when: always
    paths:
      - ${MASTER_DIR}/esp32-sr
      - ${MASTER_DIR}/data-management
      - ${MASTER_DIR}/DeepSpeech
      - ${IDF_PATH}/tools/ci
      - ${IDF_PATH}/make
      - ${IDF_PATH}/tools/tiny-test-fw
      - ${IDF_PATH}/tools/kconfig
      - ${IDF_PATH}/Kconfig
      - ${IDF_PATH}/components/nghttp/
      - ${IDF_PATH}/components/esptool_py
      - ${IDF_PATH}/components
  script:
    - mkdir -p "$MASTER_DIR"
    - cd ${MASTER_DIR}
    - git clone $DATA_RESPOSITORY
    - git clone $SPEECH_RESPOSITORY
    - git clone $AUDIO_RESPOSITORY
    - git clone $IDF_RESPOSITORY
    - cd esp-idf
    # replace submodule esp-idf to internal respository to speed cloning
    - sed -i "s%https://github.com/espressif%${GITLAB_SSH_SERVER}/idf%" .gitmodules
    - sed -ri "s%https://github.com/\w.+/%${GITLAB_SSH_SERVER}/idf/%" .gitmodules
    - git submodule update --init
    - source tools/ci/configure_ci_environment.sh

.build_template: &build_template
  stage: build
  dependencies:
   - preparation
  tags:
    - build
  variables:
    BATCH_BUILD: "1"
    V: "0"
  before_script: *do_nothing_before

build_audio:
  <<: *build_template
  image: $CI_DOCKER_REGISTRY/esp32-ci-env
  artifacts:
    when: always
    paths:
      - ${MASTER_DIR}/esp32-sr/build
  script:
    - cd ${MASTER_DIR}/esp32-sr
    - echo `pwd`
    - git submodule update --init
    - make help
    - make all
    - cd build
    - echo  "-z --flash_mode dio --flash_freq 80m --flash_size detect 0x1000 ${MASTER_DIR}/esp32-sr/build/bootloader/bootloader.bin 0x10000 ${MASTER_DIR}/esp32-sr/build/sr.bin 0x8000 ${MASTER_DIR}/esp32-sr/build/partitions.bin" > download.config

build_datamanagement:
  <<: *build_template
  script:
    - echo `pwd`

build_speech:
  <<: *build_template
  script:
    - echo `pwd`

training_step:
  stage: training
  image: $CI_DOCKER_REGISTRY/esp32-ci-env
  tags:
    - training
  dependencies:
   - preparation
   - build_audio
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/train/model_dir
  variables:
    TRAINING_PROJECT_PATH: "${MASTER_DIR}/DeepSpeech"
    DATA_PROJECT_PATH: "${MASTER_DIR}/data-management"
    KEY_WORD: "alexa"
  before_script: *do_nothing_before
  script:
    - cd $CI_PROJECT_DIR
    - pip install -r install.md
    - python Run.py -t $TRAINING_PROJECT_PATH -m "$DATA_PROJECT_PATH" -k $KEY_WORD -d "$CI_PROJECT_DIR/train/download_dir" -s "$CI_PROJECT_DIR/train/data_set" -c "$CI_PROJECT_DIR/train/checkpoint_dir" -r "$CI_PROJECT_DIR/train/model_dir"

test_audio:
  stage: test
  image: $CI_DOCKER_REGISTRY/esp32-ci-env
  tags:
    - test
  when: on_success
  dependencies:
   - training_step
   - preparation
  before_script: *do_nothing_before
  script:
    - cd ${MASTER_DIR}
    - cd ${MASTER_DIR}/esp32-sr
    # replace the original model to the training model
    - rm -rf components/nn_model/model
    - cp -r $CI_PROJECT_DIR/train/model_dir model
    - pyhton test.py -a "${MASTER_DIR}/esp32-sr/build"  -s "$CI_PROJECT_DIR/train/download_dir"

Caution:

• before_script

Create ssh key for Runner

before_script:
  # add gitlab ssh key
  - mkdir -p ~/.ssh
  - chmod 700 ~/.ssh
  - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
  - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
  - chmod 600 ~/.ssh/id_rsa
  - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config

• intranet and extranet, speed cloning

 # replace submodule esp-idf to internal respository to speed cloning
    - sed -i "s%https://github.com/espressif%${GITLAB_SSH_SERVER}/idf%" .gitmodules
    - sed -ri "s%https://github.com/\w.+/%${GITLAB_SSH_SERVER}/idf/%" .gitmodules
    - git submodule update --init
    - source tools/ci/configure_ci_environment.sh

• inherition relationship

.do_nothing_before:
  before_script: &do_nothing_before
    - echo "do nothing"

"do_nothing_before" inherited from "before_script"

Comparation:

.do_nothing_before: &do_nothing_before
  stage: build
  .
  .
  .

Just like a function can be qouted.

• globle variables

IDF_PATH: "$CI_PROJECT_DIR/WORKING_DIR/esp-idf"

It will also be used as the Runner globle variables.

• artifacts

It is the stroed cache directories will be used in other stage

• dependencies

It will use the cache directories in artifacts in previouse stage.

• image

This use docker to run, the image is the docker image where the Runner environment installed.

• solid globle variables
  
  • what will be stored here ?
    
    • where docker image located
    • GITLAB_KEY (ssh token)
    • SERVER address
    • Other respository address (download address)

This will be set in setting->CI/CD->secret variables