From 9788ccd5e1c84b8561d87188900605c7346b6df5 Mon Sep 17 00:00:00 2001 From: Evgeniy Sokolov Date: Thu, 12 Nov 2020 23:42:25 +0100 Subject: [PATCH] Github Actions for checks (#574) * github actions ci * github actions ci * github actions ci * github actions ci * github actions ci * github actions ci * no deprecation * debug * debug * debug * fix * fix * fix * fix * fix * forgotten files * fix * fix * fix * fix * oganize everything related to ci in one folder * add check * fix path * get changed files * try this env * expected codes file moved * test check * test check * test check * docker for local test ci and fix dict download * local setup and some updates * rollback * right email --- .../{update_github_pages.yml => cd.yml} | 17 ++++++++++++-- .github/workflows/ci.yml | 23 +++++++++++++++++++ .github/workflows/sort_dictionary.yml | 21 ----------------- .travis.yml | 19 --------------- CONTRIBUTING.md | 4 ++-- files/Dockerfile | 10 -------- files/ci/Dockerfile | 11 +++++++++ files/{ => ci}/check-install.sh | 12 ++++++---- files/{ => ci}/check.sh | 17 ++++++-------- files/{ => ci}/dictionary.dic | 0 files/{ => ci}/docker-compose.yml | 5 ++-- files/{ => ci}/expected_codes.csv | 0 files/{ => ci}/go/check_links/main.go | 4 ++-- files/{ => ci}/go/check_spell/main.go | 2 +- files/ci/go/go.mod | 10 ++++++++ files/ci/go/go.sum | 11 +++++++++ 16 files changed, 91 insertions(+), 75 deletions(-) rename .github/workflows/{update_github_pages.yml => cd.yml} (53%) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/sort_dictionary.yml delete mode 100644 .travis.yml delete mode 100644 files/Dockerfile create mode 100644 files/ci/Dockerfile rename files/{ => ci}/check-install.sh (80%) rename files/{ => ci}/check.sh (65%) rename files/{ => ci}/dictionary.dic (100%) rename files/{ => ci}/docker-compose.yml (53%) rename files/{ => ci}/expected_codes.csv (100%) rename files/{ => ci}/go/check_links/main.go (94%) rename files/{ => ci}/go/check_spell/main.go (96%) create mode 100644 files/ci/go/go.mod create mode 100644 files/ci/go/go.sum diff --git a/.github/workflows/update_github_pages.yml b/.github/workflows/cd.yml similarity index 53% rename from .github/workflows/update_github_pages.yml rename to .github/workflows/cd.yml index 175cc8d..a3b28b0 100644 --- a/.github/workflows/update_github_pages.yml +++ b/.github/workflows/cd.yml @@ -1,4 +1,4 @@ -name: Update github pages +name: Continuous delivery on: push: @@ -6,6 +6,19 @@ on: - master jobs: + sort_dictionary: + if: github.repository == 'ru-de/faq' + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Sort dictionary + run: | + git config --global user.email "actions@github.com" + git config --global user.name "Github Actions" + LC_ALL=ru_RU.UTF8 sort files/ci/dictionary.dic -o files/ci/dictionary.dic -f + if ! git diff HEAD --quiet; then git commit -q -am "dictionary rearrangement" && git push; fi update_github_pages: if: github.repository == 'ru-de/faq' runs-on: ubuntu-18.04 @@ -13,7 +26,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 100 - - name: update + - name: Update github pages run: | git config --global user.email "actions@github.com" git config --global user.name "Github Actions" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..aaf5ebe --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: Continious integration + +on: + pull_request: + +jobs: + spell_and_dead_links_checks: + if: github.repository == 'ru-de/faq' + runs-on: ubuntu-18.04 + steps: + - uses: actions/setup-go@v2 + with: + stable: 'false' + go-version: '1.15.4' + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Install utils + run: | + sudo bash files/ci/check-install.sh + - name: Check spell and dead links + run: | + bash files/ci/check.sh diff --git a/.github/workflows/sort_dictionary.yml b/.github/workflows/sort_dictionary.yml deleted file mode 100644 index 87d9692..0000000 --- a/.github/workflows/sort_dictionary.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Sort dictionary - -on: - push: - branches: - - master - -jobs: - sort_dictionary: - if: github.repository == 'ru-de/faq' - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - name: sort - run: | - git config --global user.email "actions@github.com" - git config --global user.name "Github Actions" - LC_ALL=ru_RU.UTF8 sort files/dictionary.dic -o files/dictionary.dic -f - if ! git diff HEAD --quiet; then git commit -q -am "dictionary rearrangement" && git push; fi diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 98d04ab..0000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -language: go -sudo: required - -dist: trusty - -git: - depth: 3 - submodules: false - -install: - - export GOPATH="${TRAVIS_BUILD_DIR}/Godeps/_workspace:$GOPATH" - - export PATH="${TRAVIS_BUILD_DIR}/Godeps/_workspace/bin:$PATH" - - sudo env PATH=$PATH GOPATH=$GOPATH bash files/check-install.sh - -script: - - bash files/check.sh - -notifications: - email: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9966873..6e9a351 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,5 +30,5 @@ После создания Pull Request измененные файлы будут проверены роботом на содержание орфографических ошибок и недоступные ссылки, дождитесь пожалуйста выполнения проверок, если какой-то check не прошел, посмотрите лог, возможно где-то допущена опечатка, использовано не словарное слово, либо ссылка на какую-то страницу недоступна. -В случае если слово, на которое ругается проверка является широко употребимым, можно внести его в словарь, для этого добавьте его в файл "files/dictionary.dic". -В случае если ссылка на самом деле доступна, но отдает нестандартный ответ, можно внести такой ответ как корректный в файл "files/known_url.csv" \ No newline at end of file +В случае если слово, на которое ругается проверка является широко употребимым, можно внести его в словарь, для этого добавьте его в файл "files/ci/dictionary.dic". +В случае если ссылка на самом деле доступна, но отдает нестандартный ответ, можно внести такой ответ как корректный в файл "files/ci/expected_codes.csv" \ No newline at end of file diff --git a/files/Dockerfile b/files/Dockerfile deleted file mode 100644 index 299b086..0000000 --- a/files/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -# check https://docs.travis-ci.com/user/common-build-problems/#troubleshooting-locally-in-a-docker-image -FROM travisci/ci-garnet:packer-1512502276-986baf0 -MAINTAINER Evgenii Sokolov - -USER travis -ENV GOPATH "/home/travis/gopath" -ENV GOROOT "/home/travis/.gimme/versions/go1.7.4.linux.amd64" -ENV PATH "$GOPATH/bin:$GOROOT/bin:$PATH" -COPY . $GOPATH/src/github.com/ru-de/faq/files -RUN cd $GOPATH/src/github.com/ru-de/faq && sudo env PATH=$PATH GOPATH=$GOPATH bash files/check-install.sh diff --git a/files/ci/Dockerfile b/files/ci/Dockerfile new file mode 100644 index 0000000..0272412 --- /dev/null +++ b/files/ci/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:18.04 +MAINTAINER Evgenii Sokolov + +RUN apt-get -yqq update && apt-get install -y wget git locales +RUN cd /tmp && wget https://golang.org/dl/go1.15.5.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.15.5.linux-amd64.tar.gz +ENV PATH "$PATH:/usr/local/go/bin:/root/go/bin" +RUN locale-gen --no-purge en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 +ENV LC_ALL "en_US.UTF-8" +COPY . /tmp/files/ci +RUN cd /tmp/files/ci && bash check-install.sh diff --git a/files/check-install.sh b/files/ci/check-install.sh similarity index 80% rename from files/check-install.sh rename to files/ci/check-install.sh index 80b9bb9..e375e09 100644 --- a/files/check-install.sh +++ b/files/ci/check-install.sh @@ -2,12 +2,13 @@ set -xe +PWD_DIR=`pwd` DIR=`dirname $0` -apt-get -yqq update && apt-get install -y hunspell hunspell-ru hunspell-en-us hunspell-de-de jq +apt-get -yqq update && apt-get install -y curl wget unzip binutils hunspell hunspell-ru hunspell-en-us hunspell-de-de jq curl -s https://extensions.libreoffice.org/en/extensions/show/russian-spellcheck-dictionary > .dict_page echo -n "https://extensions.libreoffice.org" > .current_release -strings .dict_page | grep -ozP '
  • (\n|.)*?
    • ' | grep -zoP 'href=".*?">Download' | head -1 | sed 's/href="//' | sed 's/">Download//' >> .current_release +strings .dict_page | grep -ozP '
  • (\n|.)*?
    • ' | grep -zoP -m 1 'href=".*?">Download' | head -1 | sed 's/href="//' | sed 's/">Download//' >> .current_release cat .current_release | wget -q -i - -O /tmp/dictionary.otx unzip /tmp/dictionary.otx -d /tmp cp /tmp/*.dic /usr/share/hunspell @@ -34,11 +35,12 @@ done echo "SET UTF-8" >> /tmp/dictionary.aff mv /tmp/dictionary.* /usr/share/hunspell -go get ./... go get -u github.com/ewgRa/ci-utils/cmd/diff_liner go get -u github.com/ewgRa/ci-utils/cmd/hunspell_parser go get -u github.com/ewgRa/ci-utils/cmd/github_comments_diff go get -u github.com/ewgRa/ci-utils/cmd/github_comments_send -go build -o /tmp/check_spell $DIR/go/check_spell/main.go -go build -o /tmp/check_links $DIR/go/check_links/main.go +cd $DIR/go +go build -o /tmp/check_spell check_spell/main.go +go build -o /tmp/check_links check_links/main.go +cd $PWD_DIR \ No newline at end of file diff --git a/files/check.sh b/files/ci/check.sh similarity index 65% rename from files/check.sh rename to files/ci/check.sh index 596d707..0263ad7 100644 --- a/files/check.sh +++ b/files/ci/check.sh @@ -1,17 +1,12 @@ #!/bin/bash -if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then - echo "Skip, because not a PR" - exit 0 -fi - DIR=`dirname $0` git config --global core.quotepath false -git diff HEAD^ --name-status | grep "^D" -v | sed 's/^.\t//g' | grep "\.md$" > /tmp/changed_files +PULL_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") -curl "https://github-api-bot.herokuapp.com/diff?repo=$TRAVIS_REPO_SLUG&pr=$TRAVIS_PULL_REQUEST" > /tmp/pr.diff +curl "https://github-api-bot.herokuapp.com/diff?repo=${GITHUB_REPOSITORY}&pr=${PULL_NUMBER}" > /tmp/pr.diff if [ "$?" != "0" ]; then echo "Can't get github pull request diff, probably rate limit? Try to restart CI build" @@ -23,6 +18,8 @@ cat /tmp/pr.diff | diff_liner > /tmp/pr_liner.json rm -f /tmp/comments.json touch /tmp/comments.json +curl -s -X GET -G https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PULL_NUMBER}/files | jq -r '.[] | .filename' | grep "^D" -v | sed 's/^.\t//g' | grep "\.md$" > /tmp/changed_files + while read FILE; do COMMIT=$(git log --pretty=format:"%H" -1 "$FILE"); echo "Проверка изменений в файле $FILE на опечатки... "; @@ -33,7 +30,7 @@ while read FILE; do echo "Проверка изменений в файле $FILE на недоступные ссылки... "; - /tmp/check_links -file "$FILE" -commit=$COMMIT -pr-liner /tmp/pr_liner.json -expected-codes files/expected_codes.csv >> /tmp/comments.json + /tmp/check_links -file "$FILE" -commit=$COMMIT -pr-liner /tmp/pr_liner.json -expected-codes files/ci/expected_codes.csv >> /tmp/comments.json echo done < /tmp/changed_files @@ -45,7 +42,7 @@ cat /tmp/comments_array.json EXIT_CODE=0 if [ "$(cat /tmp/comments_array.json)" != "[]" ]; then - curl "https://github-api-bot.herokuapp.com/comments?repo=$TRAVIS_REPO_SLUG&pr=$TRAVIS_PULL_REQUEST" > /tmp/pr_comments.json + curl "https://github-api-bot.herokuapp.com/comments?repo=${GITHUB_REPOSITORY}&pr=${PULL_NUMBER}" > /tmp/pr_comments.json if [ "$?" != "0" ]; then echo "Can't get github comments, probably rate limit? Try to restart ci build" @@ -54,7 +51,7 @@ if [ "$(cat /tmp/comments_array.json)" != "[]" ]; then github_comments_diff -comments /tmp/comments_array.json -exists-comments /tmp/pr_comments.json > /tmp/send_comments.json - curl -XPOST "https://github-api-bot.herokuapp.com/send_review?repo=$TRAVIS_REPO_SLUG&pr=$TRAVIS_PULL_REQUEST&body=Спасибо%20за%20PR.%20Обратите%20внимание%20на%20результаты%20автоматической%20проверки%20орфографии%20и%20ссылок" -d @/tmp/send_comments.json + curl -XPOST "https://github-api-bot.herokuapp.com/send_review?repo=${GITHUB_REPOSITORY}&pr=${PULL_NUMBER}&body=Спасибо%20за%20PR.%20Обратите%20внимание%20на%20результаты%20автоматической%20проверки%20орфографии%20и%20ссылок" -d @/tmp/send_comments.json EXIT_CODE=1 fi diff --git a/files/dictionary.dic b/files/ci/dictionary.dic similarity index 100% rename from files/dictionary.dic rename to files/ci/dictionary.dic diff --git a/files/docker-compose.yml b/files/ci/docker-compose.yml similarity index 53% rename from files/docker-compose.yml rename to files/ci/docker-compose.yml index 5c0cf1e..9d3cff6 100644 --- a/files/docker-compose.yml +++ b/files/ci/docker-compose.yml @@ -6,7 +6,6 @@ services: context: . dockerfile: Dockerfile container_name: ru_de_checker - user: travis - command: bash -l -c "cd /home/travis/ru-de-faq && bash files/check.sh" + command: bash -l -c "cd /home/ru-de-faq && bash files/ci/check.sh" volumes: - - ../.:/home/travis/ru-de-faq + - ../../.:/home/ru-de-faq diff --git a/files/expected_codes.csv b/files/ci/expected_codes.csv similarity index 100% rename from files/expected_codes.csv rename to files/ci/expected_codes.csv diff --git a/files/go/check_links/main.go b/files/ci/go/check_links/main.go similarity index 94% rename from files/go/check_links/main.go rename to files/ci/go/check_links/main.go index 8583cfc..b2505a0 100644 --- a/files/go/check_links/main.go +++ b/files/ci/go/check_links/main.go @@ -2,13 +2,13 @@ package main import ( "fmt" + "github.com/russross/blackfriday/v2" "os" "flag" "bufio" "regexp" "github.com/ewgRa/ci-utils/src/diff_liner" "github.com/ewgRa/ci-utils/src/links_checker" - "gopkg.in/russross/blackfriday.v2" "encoding/json" "github.com/google/go-github/github" ) @@ -65,7 +65,7 @@ func main() { continue } - body := fmt.Sprintf("Ссылка **%s** ... недоступна с кодом **%v**, ожидается **%v**.\nЕсли это ожидаемый ответ, внесите \"%v,%s\" в files/expected_codes.csv", link, respCode, expectedCodes, respCode, link) + body := fmt.Sprintf("Ссылка **%s** ... недоступна с кодом **%v**, ожидается **%v**.\nЕсли это ожидаемый ответ, внесите \"%v,%s\" в files/ci/expected_codes.csv", link, respCode, expectedCodes, respCode, link) comments = append(comments, &github.PullRequestComment{ Body: &body, diff --git a/files/go/check_spell/main.go b/files/ci/go/check_spell/main.go similarity index 96% rename from files/go/check_spell/main.go rename to files/ci/go/check_spell/main.go index e7c8153..9a2bc62 100644 --- a/files/go/check_spell/main.go +++ b/files/ci/go/check_spell/main.go @@ -35,7 +35,7 @@ func main() { continue } - body := fmt.Sprintf("Возможная ошибка в слове \"**%s**\".\n Варианты правильного написания \"**%s**\".\nЕсли слово \"%s\" является правильным, добавьте его в files/dictionary.dic", resp.Word, resp.Alternative, resp.Word) + body := fmt.Sprintf("Возможная ошибка в слове \"**%s**\".\n Варианты правильного написания \"**%s**\".\nЕсли слово \"%s\" является правильным, добавьте его в files/ci/dictionary.dic", resp.Word, resp.Alternative, resp.Word) comments = append(comments, &github.PullRequestComment{ Body: &body, diff --git a/files/ci/go/go.mod b/files/ci/go/go.mod new file mode 100644 index 0000000..3eda43e --- /dev/null +++ b/files/ci/go/go.mod @@ -0,0 +1,10 @@ +module github.com/ru-de/faq + +go 1.15 + +require ( + github.com/ewgRa/ci-utils v0.0.0-20180607200228-15dd1fde41f9 + github.com/google/go-github v17.0.0+incompatible + github.com/google/go-querystring v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect +) diff --git a/files/ci/go/go.sum b/files/ci/go/go.sum new file mode 100644 index 0000000..422f0e2 --- /dev/null +++ b/files/ci/go/go.sum @@ -0,0 +1,11 @@ +github.com/ewgRa/ci-utils v0.0.0-20180607200228-15dd1fde41f9 h1:G+RMure2Z4ebQs/jCfeeqFB4k73bO7H9FJN4B/DX+vw= +github.com/ewgRa/ci-utils v0.0.0-20180607200228-15dd1fde41f9/go.mod h1:VOwWqySz9ACgsDQRgWcJmOQL0HYyugfTvP1adEmGWbE= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v1.6.0-pre.1 h1:APCbYYLyOoAWCgZNdHFH3o9uon5Z+N7gp6pThPjuvwI= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=