小更一下

src/main/java/net/lensfrex/oj/collector/Collector.java

@@ -5,13 +5,12 @@ import net.lensfrex.oj.collector.data.QuestionDetail;
 import net.lensfrex.oj.collector.utils.IOUtil;
 import net.lensfrex.oj.collector.utils.NetworkUtil;
 import net.lensfrex.oj.collector.utils.Random;
-import org.apache.http.client.utils.URIBuilder;
 
 import java.io.IOException;
-import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Map;
 
 public class Collector {
 
@@ -21,68 +20,87 @@ public class Collector {
 
     private static final String INDEX_REQUEST_BODY = IOUtil.inputStreamToString(Record.class.getResourceAsStream("/index_request_body.txt"), StandardCharsets.UTF_8);
 
+    // leetcode每次最大只能获取100条。就算设置1000每次也只能获取到100条
     private static final int PAGE_LIMIT = 100;
 
-    private String generatePageRequestUrl(String baseUrl, int offset, int limit, int page) throws URISyntaxException {
+    public ArrayList<QuestionDetail> collectAllQuestion() {
-        URIBuilder uriBuilder = new URIBuilder(baseUrl);
+        ArrayList<QuestionDetail> QuestionDetails = new ArrayList<>();
-        uriBuilder.addParameter("paging", "true");
+        try {
-        uriBuilder.addParameter("offset", String.valueOf(offset));
+            // 先拿一条记录看看有多少题目
-        uriBuilder.addParameter("limit", String.valueOf(limit));
+            String json = NetworkUtil.post(API_BASE, String.format(INDEX_REQUEST_BODY, 0, 1), headers);
-        uriBuilder.addParameter("page", String.valueOf(page));
 
-        return uriBuilder.toString();
+            int totalResult = JsonParser.parseString(json).getAsJsonObject()
-    }
+                    .getAsJsonObject("data")
+                    .getAsJsonObject("problemsetQuestionList")
+                    .getAsJsonPrimitive("total").getAsInt();
 
-    private HashMap<String, String> getHeaders() {
+            int page = (totalResult / PAGE_LIMIT) + (totalResult % PAGE_LIMIT == 0 ? 0 : 1);
-        HashMap<String, String> headers = new HashMap<>();
 
-        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.66 Safari/537.36 Edg/103.0.1264.44");
+            System.out.println("Total results: " + totalResult);
-        headers.put("Host", "leetcode.cn");
+            System.out.println("Page Limit: " + PAGE_LIMIT);
-        headers.put("Origin", "https://leetcode.cn");
+            System.out.println("Total pages: " + page);
-        headers.put("Accept-Encoding", "gzip, deflate, br");
-        headers.put("Content-Type", "application/json");
 
-        return headers;
+            // Fetch all question list
+            ArrayList<String> questionNames = this.fetchQuestionList(page);
+
+            System.out.println("Total result: " + questionNames.size());
+
+            // Fetch all question details
+
+            for (String questionTitleSlug : questionNames) {
+                QuestionDetail detail = fetchQuestionDetail(questionTitleSlug);
+
+                record.writeToFile(
+                        "D:\\ojs-leetcode",
+                        String.valueOf(detail.getId()),
+                        detail.getChineseTitle() == null ? detail.getTitle() : detail.getChineseTitle(),
+                        record.fillText(detail));
+
+                Thread.sleep(random.getRandomNumber(256));
             }
 
+        } catch (IOException | InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+
+        System.out.println("Finish.");
+
+        return QuestionDetails;
+    }
 //
-//    private QuestionDetail[] getResults(String url, Map<String, String> headers) throws IOException {
+//    private String generatePageRequestUrl(String baseUrl, int offset, int limit, int page) throws URISyntaxException {
-//        String json = NetworkUtil.get(url, headers);
+//        URIBuilder uriBuilder = new URIBuilder(baseUrl);
-//
+//        uriBuilder.addParameter("paging", "true");
-//        JsonArray resultJsonArray = JsonParser.parseString(json).getAsJsonObject()
+//        uriBuilder.addParameter("offset", String.valueOf(offset));
-//                .getAsJsonObject("data")
+//        uriBuilder.addParameter("limit", String.valueOf(limit));
-//                .getAsJsonArray("results");
+//        uriBuilder.addParameter("page", String.valueOf(page));
 //
-//        return new Gson().fromJson(resultJsonArray, QuestionDetail[].class);
+//        return uriBuilder.toString();
 //    }
 
     private final Record record = new Record();
     private static final Random random = new Random();
 
-    // 起始位置
+    private static HashMap<String, String> setHeaders() {
-    private static final int START_AT = 0;
+        HashMap<String, String> headers = new HashMap<>();
-
-    public ArrayList<QuestionDetail> collectAllQuestion() {
-        ArrayList<QuestionDetail> QuestionDetails = new ArrayList<>();
-        try {
-            HashMap<String, String> headers = getHeaders();
-
-            String json = NetworkUtil.post(API_BASE, String.format(INDEX_REQUEST_BODY, 0, 1), headers);
 
-            int totalResult = JsonParser.parseString(json).getAsJsonObject()
+        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.66 Safari/537.36 Edg/103.0.1264.44");
-                    .getAsJsonObject("data")
+        headers.put("Host", "leetcode.cn");
-                    .getAsJsonObject("problemsetQuestionList")
+        headers.put("Origin", "https://leetcode.cn");
-                    .getAsJsonPrimitive("total").getAsInt();
+        headers.put("Accept-Encoding", "gzip, deflate, br");
+        headers.put("Content-Type", "application/json");
 
-            int page = (totalResult / PAGE_LIMIT) + (totalResult % PAGE_LIMIT == 0 ? 0 : 1);
+        return headers;
+    }
 
-            ArrayList<String> questionNames = new ArrayList<>(totalResult);
+    private static final Map<String, String> headers = setHeaders();
 
-            System.out.println("Total results: " + totalResult);
+    // 起始位置
-            System.out.println("Page Limit: " + PAGE_LIMIT);
+    private static final int START_AT = 0;
-            System.out.println("Total pages: " + page);
 
-            // Fetch all question list
+    private ArrayList<String> fetchQuestionList(int page) throws IOException {
+        ArrayList<String> questions = new ArrayList<>(page * PAGE_LIMIT);
+        String json = "";
         for (int i = 1; i <= page; i++) {
             int offset = (i - 1) * PAGE_LIMIT;
             System.out.println("\n------------------------------------------------------");
@@ -98,40 +116,26 @@ public class Collector {
                             .getAsJsonArray("questions");
 
             for (JsonElement jsonElement : questionJsonArray) {
-                    questionNames.add(jsonElement.getAsJsonObject().get("titleSlug").getAsString());
+                questions.add(jsonElement.getAsJsonObject().get("titleSlug").getAsString());
             }
 
             System.out.println("Got result " + questionJsonArray.size());
         }
 
-            System.out.println("Total result: " + questionNames.size());
+        return questions;
+    }
 
-            // Fetch all question details
+    private QuestionDetail fetchQuestionDetail(String name) throws IOException {
-            QuestionDetail detail = null;
+        System.out.println("\n------------------------------------------------------");
-            for (String questionTitleSlug : questionNames) {
+        System.out.println("Getting question: " + name);
-                System.out.println("Getting question: " + questionTitleSlug);
+
-                json = NetworkUtil.post(API_BASE, String.format(DETAILS_REQUEST_BODY, questionTitleSlug), headers);
+        String json = NetworkUtil.post(API_BASE, String.format(DETAILS_REQUEST_BODY, name), headers);
 
         JsonObject question =
                 JsonParser.parseString(json).getAsJsonObject()
                         .getAsJsonObject("data")
                         .getAsJsonObject("question");
 
-                detail = new Gson().fromJson(question, QuestionDetail.class);
+        return new Gson().fromJson(question, QuestionDetail.class);
-                record.writeToFile(
-                        "D:\\ojs-leetcode",
-                        String.valueOf(detail.getId()),
-                        detail.getChineseTitle() == null ? detail.getTitle() : detail.getChineseTitle(),
-                        record.fillText(detail));
-
-                Thread.sleep(random.getRandomNumber(120));
-            }
-
-        } catch (IOException | InterruptedException e) {
-            throw new RuntimeException(e);
-        }
-
-        System.out.println("Finish.");
-        return QuestionDetails;
     }
 }

src/main/java/net/lensfrex/oj/collector/Main.java

@@ -24,14 +24,6 @@ public class Main {
     private static final String location = "D:\\ojs";
 
     private void run() throws URISyntaxException, IOException {
-        Record record = new Record();
+        new Collector().collectAllQuestion();
-        ArrayList<QuestionDetail> questionResults = new Collector().collectAllQuestion();
-//
-//        String context = "";
-//        for (QuestionDetail questionResult : questionResults) {
-//            context = record.fillText(questionResult);
-//
-//            record.writeToFile(location, String.valueOf(questionResult.getId()), questionResult.getChineseTitle(), context);
-//        }
     }
 }

src/main/java/net/lensfrex/oj/collector/Record.java

@@ -60,6 +60,8 @@ public class Record {
         title = filenameFilter(title);
 
         File file = new File(String.format("%s/%s - %s.md", location, id, title));
+        file.getParentFile().mkdirs();
+
         System.out.println("Will write to: " + file.getPath());
         try {
             IOUtil.writeFile(context.getBytes(StandardCharsets.UTF_8), file);

src/main/java/net/lensfrex/oj/collector/utils/ParamFiller.java

@@ -20,6 +20,7 @@ public class ParamFiller {
 
             if (value != null) {
                 // 直接正则替换遇到$会报错
+                // 后面会改一改
                 value = value.replace("$", "＄");
             }
             matcher.appendReplacement(stringBuffer, value == null ? "" : value);