diff --git a/.travis.yml b/.travis.yml index 652d04c..8f49600 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,8 @@ install: - sudo env PATH=$PATH GOPATH=$GOPATH bash files/check-install.sh script: - - bash files/check.sh - bash files/run_on_master.sh + - bash files/check.sh notifications: email: false diff --git a/files/check-install.sh b/files/check-install.sh index a58329e..09a50aa 100644 --- a/files/check-install.sh +++ b/files/check-install.sh @@ -2,7 +2,9 @@ set -xe -apt-get -yqq update && apt-get install -y hunspell hunspell-ru hunspell-en-us hunspell-de-de +DIR=`dirname $0` + +apt-get -yqq update && apt-get install -y hunspell hunspell-ru hunspell-en-us hunspell-de-de jq curl -s https://extensions.libreoffice.org/extensions/russian-spellcheck-dictionary.-based-on-works-of-aot-group > .dict_page cat .dict_page | grep -oP " .current_release echo -n $(cat .current_release) > .current_release @@ -27,4 +29,17 @@ for dic in $dicList cat /usr/share/hunspell/$dic.aff | iconv --from ISO8859-1 --to UTF-8 | sed 's/SET ISO8859-1/SET UTF-8/' > /usr/share/hunspell/$dic-utf8.aff done -go get -u github.com/russross/blackfriday-tool +go get ./... +go get -u github.com/ewgRa/ci-utils/utils/diff_liner +go get -u github.com/ewgRa/ci-utils/utils/hunspell_parser +go get -u github.com/ewgRa/ci-utils/utils/github_comments_diff +go get -u github.com/ewgRa/ci-utils/utils/github_comments_send + +go build -o /tmp/check_spell $DIR/go/check_spell/main.go +go build -o /tmp/check_links $DIR/go/check_links/main.go + +(cat $DIR/dictionary.dic; echo) | sed '/^$/d' | wc -l > /tmp/dictionary.dic +(cat $DIR/dictionary.dic; echo) | sed '/^$/d' >> /tmp/dictionary.dic + +echo "SET UTF-8" >> /tmp/dictionary.aff +sudo mv /tmp/dictionary.* /usr/share/hunspell diff --git a/files/check.sh b/files/check.sh index 0e291c1..be0dd87 100644 --- a/files/check.sh +++ b/files/check.sh @@ -1,66 +1,45 @@ #!/bin/bash DIR=`dirname $0` -EXIT_CODE=0 git config --global core.quotepath false -(cat $DIR/dictionary.dic; echo) | sed '/^$/d' | wc -l > /tmp/dictionary.dic -(cat $DIR/dictionary.dic; echo) | sed '/^$/d' >> /tmp/dictionary.dic - -echo "SET UTF-8" >> /tmp/dictionary.aff -sudo mv /tmp/dictionary.* /usr/share/hunspell - git diff HEAD^ --name-status | grep "^D" -v | sed 's/^.\t//g' | grep "\.md$" > /tmp/changed_files -go build -o /tmp/spell-checker $DIR/spell-checker.go +curl -sH "Accept: application/vnd.github.v3.diff.json" https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/$TRAVIS_PULL_REQUEST > /tmp/pr.diff +cat /tmp/pr.diff | diff_liner > /tmp/pr_liner.json + +rm -f /tmp/comments.json +touch /tmp/comments.json while read FILE; do - echo -n "Проверка файла $FILE на опечатки... "; + COMMIT=$(git log --pretty=format:"%H" -1 "$FILE"); + echo "Проверка изменений в файле $FILE на опечатки... "; - OUTPUT=$(cat "$FILE" | sed 's/https\?:[^ ]*//g' | sed "s/[(][^)]*\.md[)]//g" | sed "s/[(]files[^)]*[)]//g" | hunspell -d dictionary,russian-aot-utf8,ru_RU,de_DE-utf8,en_US-utf8 | /tmp/spell-checker); - OUTPUT_EXIT_CODE=$? + cat "$FILE" | sed 's/https\?:[^ ]*//g' | sed "s/[(][^)]*\.md[)]//g" | sed "s/[(]files[^)]*[)]//g" | hunspell -d dictionary,russian-aot-utf8,ru_RU,de_DE-utf8,en_US-utf8 > /tmp/hunspell.out + cat /tmp/hunspell.out | hunspell_parser > /tmp/hunspell_parsed.json + /tmp/check_spell -file "$FILE" -commit=$COMMIT -pr-liner /tmp/pr_liner.json -hunspell-parsed-file /tmp/hunspell_parsed.json >> /tmp/comments.json - if [ $OUTPUT_EXIT_CODE -ne 0 ]; then - EXIT_CODE=1 - echo "ошибка"; - echo "$OUTPUT"; - else - echo "пройдена"; - fi + echo "Проверка изменений в файле $FILE на недоступные ссылки... "; - rm -f /tmp/file.html - blackfriday-tool "$FILE" /tmp/file.html - - if [ -f "/tmp/file.html" ]; then - grep -Po '(?<=href=")http[^"]*(?=")' "/tmp/file.html" > /tmp/links - - if [ -s /tmp/links ]; then - echo "Проверка файла $FILE на недоступные ссылки... "; - - while read LINK; do - REGEXP_LINK=$(echo $LINK | sed 's/[]\.|$(){}?+*^[]/\\&/g') - LINK=$(echo "$LINK" | sed -e 's/\[/\\\[/g' -e 's/\]/\\\]/g' -e 's/\&/\&/g') - status=$(curl --insecure -XGET -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" -m 10 -L -s --head -w %{http_code} $LINK -o /dev/null) - expectedStatus=$(grep -oP "[^,]+,$REGEXP_LINK$" $DIR/known_url.csv | cut -d',' -f1) - - if [ -z "$expectedStatus" ]; then - expectedStatus="200" - fi - - if [ "$status" != "$expectedStatus" -a "$status" != "200" ]; then - EXIT_CODE=1 - echo "Ссылка $LINK ... недоступна с кодом $status, ожидается $expectedStatus"; - echo - fi - - done < /tmp/links - - echo - fi - fi + /tmp/check_links -file "$FILE" -commit=$COMMIT -pr-liner /tmp/pr_liner.json -expected-codes files/expected_codes.csv >> /tmp/comments.json echo done < /tmp/changed_files +jq -s '[.[][]]' /tmp/comments.json > /tmp/comments_array.json + +curl -s https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/$TRAVIS_PULL_REQUEST/comments > /tmp/pr_comments.json + +github_comments_diff -comments /tmp/comments_array.json -exists-comments /tmp/pr_comments.json > /tmp/send_comments.json + +OUTPUT=$(cat /tmp/comments_array.json | grep "\[]"); +EXIT_CODE=$? + +if [ $EXIT_CODE -ne 0 ]; then + github_comments_send -file /tmp/send_comments.json -repo $TRAVIS_REPO_SLUG -pr $TRAVIS_PULL_REQUEST +fi + +cat /tmp/comments_array.json + exit $EXIT_CODE diff --git a/files/known_url.csv b/files/expected_codes.csv similarity index 100% rename from files/known_url.csv rename to files/expected_codes.csv diff --git a/files/go/check_links/main.go b/files/go/check_links/main.go new file mode 100644 index 0000000..8583cfc --- /dev/null +++ b/files/go/check_links/main.go @@ -0,0 +1,93 @@ +package main + +import ( + "fmt" + "os" + "flag" + "bufio" + "regexp" + "github.com/ewgRa/ci-utils/src/diff_liner" + "github.com/ewgRa/ci-utils/src/links_checker" + "gopkg.in/russross/blackfriday.v2" + "encoding/json" + "github.com/google/go-github/github" +) + +func main() { + prLiner := flag.String("pr-liner", "", "Pull request liner") + fileName := flag.String("file", "", "Hunspell parsed file name") + commit := flag.String("commit", "", "Commit") + expectedCodesFile := flag.String("expected-codes", "", "Expected codes file name") + flag.Parse() + + if *prLiner == "" || *fileName == "" || *commit == "" || *expectedCodesFile == "" { + flag.Usage() + os.Exit(1) + } + + linkRegexp := regexp.MustCompile("href=\"(http[^\"]*)\"") + + linerResp := diff_liner.ReadLinerResponse(*prLiner) + checker := links_checker.NewChecker(*expectedCodesFile) + + line := 0 + + file, err := os.Open(*fileName) + + if err != nil { + panic(err) + } + + defer file.Close() + + scanner := bufio.NewScanner(file) + + var comments []*github.PullRequestComment + + for scanner.Scan() { + line++ + + prLine := linerResp.GetDiffLine(*fileName, line) + + if prLine == 0 { + continue + } + + output := blackfriday.Run(scanner.Bytes()) + matches := linkRegexp.FindAllStringSubmatch(string(output), -1) + + for _, match := range matches { + link := match[1] + + ok, respCode, expectedCodes := checker.Check(link) + + if ok { + continue + } + + body := fmt.Sprintf("Ссылка **%s** ... недоступна с кодом **%v**, ожидается **%v**.\nЕсли это ожидаемый ответ, внесите \"%v,%s\" в files/expected_codes.csv", link, respCode, expectedCodes, respCode, link) + + comments = append(comments, &github.PullRequestComment{ + Body: &body, + CommitID: commit, + Path: fileName, + Position: &prLine, + }) + } + } + + if err := scanner.Err(); err != nil { + panic(err) + + } + + if len(comments) > 0 { + jsonData, err := json.Marshal(comments) + + if err != nil { + panic(err) + } + + fmt.Println(string(jsonData)) + } +} diff --git a/files/go/check_spell/main.go b/files/go/check_spell/main.go new file mode 100644 index 0000000..a1ab210 --- /dev/null +++ b/files/go/check_spell/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + "os" + "github.com/ewgRa/ci-utils/src/diff_liner" + "github.com/ewgRa/ci-utils/src/hunspell_parser" + "github.com/google/go-github/github" + "flag" + "encoding/json" +) + +func main() { + prLiner := flag.String("pr-liner", "", "Pull request liner") + hunspellParsedFile := flag.String("hunspell-parsed-file", "", "Hunspell parsed file name") + file := flag.String("file", "", "Hunspell parsed file name") + commit := flag.String("commit", "", "Commit") + flag.Parse() + + if *prLiner == "" || *hunspellParsedFile == "" || *file == "" || *commit == "" { + flag.Usage() + os.Exit(1) + } + + linerResp := diff_liner.ReadLinerResponse(*prLiner) + + hunspellParsedResp := hunspell_parser.ReadHunspellParserResponse(*hunspellParsedFile) + + var comments []*github.PullRequestComment + + for _, resp := range hunspellParsedResp { + prLine := linerResp.GetDiffLine(*file, resp.Line) + + if prLine == 0 { + continue + } + + body := fmt.Sprintf("Возможная ошибка в слове \"**%s**\".\n Варианты правильного написания \"**%s**\".\nЕсли слово \"%s\" является правильным, добавьте его в files/dictionary.dic", resp.Word, resp.Alternative, resp.Word) + + comments = append(comments, &github.PullRequestComment{ + Body: &body, + CommitID: commit, + Path: file, + Position: &prLine, + }) + } + + if len(comments) > 0 { + jsonData, err := json.Marshal(comments) + + if err != nil { + panic(err) + } + + fmt.Println(string(jsonData)) + } +} diff --git a/files/spell-checker.go b/files/spell-checker.go deleted file mode 100644 index 7be4f5c..0000000 --- a/files/spell-checker.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "fmt" - "os" - "bufio" - "regexp" - "strings" -) - - -type TypeResult struct { - results []Result -} - -type Result struct { - line int - word string -} - -func main() { - scanner := bufio.NewScanner(os.Stdin) - types := parseHunspellOutput(scanner); - - _, ok := types["&"] - - if ok { - var dropCol = regexp.MustCompile(`^([^ ]+) \d+ \d+:(.*)$`) - var minimumWord = regexp.MustCompile(`^[^ ]{3}`) - exitCode := 0 - - for _, result := range types["&"].results { - if minimumWord.MatchString(result.word) { - exitCode = 1 - fmt.Println("Строка " + fmt.Sprintf("%v", result.line) + ": " + dropCol.ReplaceAllString(result.word, "$1 >$2")) - } - } - - os.Exit(exitCode) - } -} - -func parseHunspellOutput(scanner *bufio.Scanner) map[string]*TypeResult { - line := 1; - types := make(map[string]*TypeResult) - - scanner.Scan() - - for scanner.Scan() { - text := scanner.Text() - - if text == "" { - line++; - } else { - resultType := text[0:1] - - typeResult, ok := types[resultType] - - if !ok { - typeResult = &TypeResult{} - types[resultType] = typeResult - } - - typeResult.results = append(typeResult.results, Result{line: line, word: strings.Trim(text[1:], " ")}) - } - } - - return types -}