feat:dcard 爬虫

.gitignore

@@ -16,4 +16,5 @@ reptile_data/**/*.json
 *.app
 *.exe
 *.deb
-browser/*chrome*
\ No newline at end of file
+browser/*chrome*
+browser/**/chromedriver
\ No newline at end of file

utils/createBrowserDriver.py

@@ -3,18 +3,23 @@ import platform
 
 from selenium import webdriver
 # ---------------   selenium 依赖 start ----------------
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from webdriver_manager.chrome import ChromeDriverManager
-from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
 from selenium.webdriver.chrome.service import Service as ChromeService
-from selenium.webdriver.chrome.service import Service as ChromiumService
-from webdriver_manager.chrome import ChromeDriverManager
-from webdriver_manager.core.utils import ChromeType
+from selenium.webdriver.chromium.service import ChromiumService
 from selenium.webdriver.firefox.service import Service as FirefoxService
-from webdriver_manager.firefox import GeckoDriverManager
+from selenium.webdriver.edge.service import Service as EdgeService
+from selenium.webdriver.ie.service import Service as IeService
+from selenium.webdriver.safari.service import Service as SafariService
+
+from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
 # ---------------   selenium 依赖 end ----------------
 
+# ---------------  webdriver_manager 依赖 start  -----------------
+from webdriver_manager.chrome import ChromeDriverManager
+from webdriver_manager.firefox import GeckoDriverManager
+from webdriver_manager.microsoft import EdgeChromiumDriverManager
+from webdriver_manager.opera import OperaDriverManager
+# ---------------  webdriver_manager 依赖 end  -----------------
+
 from utils.index import get_screen_resolution
 
 # from mozprofile import FirefoxProfile
@@ -23,7 +28,7 @@ from utils.index import get_screen_resolution
 '''
 
 
-def create(option=None, using_user_data=True, web_browser="firefox"):
+def create(option=None, using_user_data=True, web_browser="chrome_test"):
     """
 
     :param web_browser:
@@ -41,8 +46,10 @@ def create(option=None, using_user_data=True, web_browser="firefox"):
     user_data_dir = os.path.join(os.path.abspath("../"), 'network-assets-reptile', 'user_data')
     if web_browser == "firefox":
         options = webdriver.FirefoxOptions()
-    else:
+    elif web_browser == "chrome" or web_browser == "chrome_test":
         options = webdriver.ChromeOptions()
+    elif web_browser == "edge":
+        options = webdriver.EdgeOptions()
 
     if option is not None:
         for value in option:
@@ -57,10 +64,12 @@ def create(option=None, using_user_data=True, web_browser="firefox"):
             # options.add_argument(f'--user-data-dir={user_data_dir}')
         elif web_browser == "chrome":
             options.add_argument(f'--user-data-dir={user_data_dir}')
-        elif web_browser == "chromium":
+        elif web_browser == "chrome_test":
             options.add_argument(f'--user-data-dir={user_data_dir}')
         elif web_browser == "chrome_test":
             options.add_argument(f'--user-data-dir={user_data_dir}')
+        elif web_browser == "edge":
+            options.add_argument(f'--user-data-dir={user_data_dir}')
         else:
             print("")
 
@@ -85,22 +94,36 @@ def create(option=None, using_user_data=True, web_browser="firefox"):
         web_browser = webdriver.Chrome(options=options, service=ChromeService(ChromeDriverManager().install()))
     elif web_browser == "chrome_test":
         binary_location = ""
+        webdriver_location = ""
         if platform.system() == "Windows":
-            binary_location = os.path.join(os.path.abspath("../"), 'network-assets-reptile', 'browser', "chrome-win64",
-                                           "chrome")
+            binary_location = os.path.join(os.path.abspath("../"), 'network-assets-reptile', 'browser', "chrome_win64",
+                                           "chrome.exe")
+            webdriver_location = os.path.join(os.path.abspath("../"), 'network-assets-reptile', 'browser',"web-driver","chromedriver_win32",
+                                           "chromedriver.exe")
         elif platform.system() == "Linux":
             binary_location = os.path.join(os.path.abspath("../"), 'network-assets-reptile', 'browser',
                                            "chrome-linux64", "chrome")
+            webdriver_location = os.path.join(os.path.abspath("../"), 'network-assets-reptile', 'browser',"web-driver","chromedriver_linux64",
+                                           "chromedriver")
         elif platform.system() == "Darwin":
             binary_location = os.path.join(os.path.abspath("../"), 'network-assets-reptile', 'browser',
-                                           "chrome-mac-x64", "chrome")
+                                           "chrome-mac-x64", "chrome.app")
+            webdriver_location = os.path.join(os.path.abspath("../"), 'network-assets-reptile', 'browser',"web-driver","chromedriver_mac64",
+                                           "chromedriver")
         else:
             print("")
         # 指定浏览器路径
-        print(binary_location)
+        # print(binary_location)
+        # 指定浏览器路径
         options.binary_location = binary_location
-        browser = webdriver.Chrome(options=options, service=ChromiumService(
-            ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install()))
+        # options.browser_version = "114"
+        # 设置驱动二进制可执行文件路径
+        # service = ChromeService(executable_path=webdriver_location)
+        service = ChromeService(executable_path=webdriver_location)
+        # service=service=ChromeService(ChromeDriverManager().install())
+        browser = webdriver.Chrome(options=options,service=service)
+    elif web_browser == "edge":
+        browser = webdriver.Edge(options=options,service=EdgeService(EdgeChromiumDriverManager().install()))
     else:
         print("")