cackle-import-comments

1
package main
2

3
import (
4
	"encoding/json"
5
	"fmt"
6
	"io/ioutil"
7
	"log"
8
	"net/http"
9
	"os"
10
	"strconv"
11
	"time"
12

13
	"github.com/joho/godotenv"
14
)
15

16
type Channel struct {
17
	Id      int    `json:"id"`
18
	Channel string `json:"channel"`
19
	Url     string `json:"url"`
20
	Title   string `json:"title"`
21
	Created int64  `json:"created"`
22
	Count   int    `json:"count"` //вообще там всегда вроде 0, но добавил для совместимости
23
}
24

25
type Author struct {
26
	ID       int    `json:"id"`
27
	Name     string `json:"name"`
28
	Avatar   string `json:"avatar"`
29
	Www      string `json:"www"`
30
	Provider string `json:"provider"`
31
	OpenID   string `json:"openId"`
32
	Verify   bool   `json:"verify"`
33
	Notify   bool   `json:"notify"`
34
}
35

36
type Comment struct {
37
	ID       int     `json:"id"`
38
	SiteID   int     `json:"siteId"`
39
	ParentID int     `json:"parentId"`
40
	Path     []int   `json:"path"`
41
	Message  string  `json:"message"`
42
	Rating   int     `json:"rating"`
43
	Status   string  `json:"status"`
44
	Created  int64   `json:"created"`
45
	Author   Author  `json:"author"`
46
	Chan     Channel `json:"chan"`
47
	IP       string  `json:"ip"`
48
	Modified string  `json:"modified"`
49
}
50

51
func main() {
52
	log.Println("Start import comments from cackle.me...")
53
	err := godotenv.Load(".env")
54
	if err != nil {
55
		log.Fatal("Error loading .env file")
56
	}
57
	log.Printf("Get channels from widget ID: %s", os.Getenv("ID"))
58
	timeout, err := strconv.Atoi(os.Getenv("TIMEOUT"))
59
	if err != nil || timeout <= 0 {
60
		timeout = 5
61
	}
62
	// читаем каналы (по сути это список страниц, к которым есть комментарии)
63
	apiUrlChannels := fmt.Sprintf("http://cackle.me/api/3.0/comment/chan/list.json?id=%s&siteApiKey=%s&accountApiKey=%s",
64
		os.Getenv("ID"), os.Getenv("SITE_API_KEY"), os.Getenv("ACCOUNT_API_KEY"))
65
	//log.Println(apiUrlChannels)
66
	page := 0
67
	var allChannels []Channel
68
	for { //цикл по получению списка каналов, следуя рекомендации сервиса запросы делаются с паузами и по 100 шт
69
		count, channels, err := getChannels(apiUrlChannels, page)
70
		if err != nil {
71
			log.Fatal(err)
72
		}
73
		log.Printf("Get channels, iteration %d, count %d\n", page+1, len(channels))
74
		page++
75
		allChannels = append(allChannels, channels...)
76
		if count < 100 {
77
			break
78
		}
79
		time.Sleep(time.Duration(timeout) * time.Second)
80
	}
81
	log.Printf("All channels count %d\n", len(allChannels))
82
	log.Printf("Saving all channels to file...")
83

84
	file, err := json.MarshalIndent(allChannels, "", " ")
85
	if err != nil {
86
		log.Fatal(err)
87
	}
88
	err = ioutil.WriteFile("channels.json", file, 0644)
89
	if err != nil {
90
		log.Fatal(err)
91
	}
92
	log.Printf("Done\n")
93
	//читаем комментарии
94
	log.Println("Getting comments...")
95
	apiUrlComments := fmt.Sprintf("http://cackle.me/api/3.0/comment/list.json?id=%s&siteApiKey=%s&accountApiKey=%s",
96
		os.Getenv("ID"), os.Getenv("SITE_API_KEY"), os.Getenv("ACCOUNT_API_KEY"))
97
	var allComments []Comment
98
	for index, channel := range allChannels {
99
		log.Printf("Get comments from channel id %d (%d from %d),\n URL: %s\n Title: %s\n", channel.Id, index, len(allChannels), channel.Url, channel.Title)
100
		commentId := 0
101
		currentCount := 0
102
		for {
103
			time.Sleep(time.Duration(timeout) * time.Second)
104
			count, comments, err := getComments(apiUrlComments, channel.Channel, commentId)
105
			log.Println("Get block with count: ", count)
106
			currentCount += count
107
			if err != nil {
108
				log.Println(err)
109
				break
110
			} else {
111
				allComments = append(allComments, comments...)
112
				if count < 100 {
113
					commentId = 0
114
				} else {
115
					commentId = comments[99].ID
116
				}
117
			}
118
			if commentId == 0 {
119
				break
120
			}
121
		}
122
		log.Printf("Count comments: %d", currentCount)
123
		currentCount = 0
124
	}
125
	log.Println("Done...")
126
	log.Printf("All comments count %d\n", len(allComments))
127
	log.Printf("Saving all comments to file...")
128
	file, err = json.MarshalIndent(allComments, "", " ")
129
	if err != nil {
130
		log.Fatal(err)
131
	}
132
	err = ioutil.WriteFile("comments.json", file, 0644)
133
	if err != nil {
134
		log.Fatal(err)
135
	}
136
	log.Println("Done...")
137
}
138

139
func getChannels(baseApiUrl string, page int) (count int, channels []Channel, err error) {
140
	if page > 0 {
141
		baseApiUrl = baseApiUrl + fmt.Sprintf("&page=%d", page)
142
	}
143
	response, err := http.Get(baseApiUrl)
144
	if err != nil {
145
		return 0, nil, err
146
	}
147
	var result map[string][]Channel
148
	err = json.NewDecoder(response.Body).Decode(&result)
149
	if err != nil {
150
		return 0, nil, err
151
	}
152
	count = len(result["chans"])
153
	channels = result["chans"]
154
	err = nil
155
	return
156
}
157

158
func getComments(baseApiUrl string, channel string, commentId int) (count int, comments []Comment, err error) {
159
	baseApiUrl = baseApiUrl + fmt.Sprintf("&chan=%s", channel) + fmt.Sprintf("&commentId=%s", commentId)
160

161
	response, err := http.Get(baseApiUrl)
162
	if err != nil {
163
		return 0, nil, err
164
	}
165
	var result map[string][]Comment
166
	err = json.NewDecoder(response.Body).Decode(&result)
167
	if err != nil {
168
		return 0, nil, err
169
	}
170
	count = len(result["comments"])
171
	comments = result["comments"]
172
	err = nil
173
	return
174
}
175