节选 Next 按钮逻辑的 槽函数
def next_schedule(self): def start_and_search(): self.log.debug('===--→ -*- searching') self.next_btn.setText('Next') keyword = self.searchinput.text()[6:].strip() index = self.chooseBox.currentIndex() if self.nextclickCnt == 0: # 从section步 回parse步 的话以免重开 self.bThread = WorkThread(self) def crawl_btn(text): if len(text) > 5: self.crawl_btn.setEnabled(self.step_recv()=='parse section') self.next_btn.setDisabled(self.crawl_btn.isEnabled()) self.chooseinput.textChanged.connect(crawl_btn) self.p = Process(target=crawl_what, args=(index, self.print_Q, self.bar, self.current_Q, self.step_Q)) self.bThread.print_signal.connect(self.textbrowser_load) self.bThread.item_count_signal.connect(self.processbar_load) self.bThread.finishSignal.connect(self.crawl_end) self.p.start() self.bThread.start() self.log.info(f'-*-*- Background thread starting') self.chooseBox.setDisabled(True) self.params_send({'keyword':keyword}) self.log.debug(f'website_index:[{index}], keyword [{keyword}] success ') def _next(): self.log.debug('===--→ nexting') self.judge_retry() # 非retry的时候先把retry=Flase解锁spider的下一步 choose = judge_input(self.chooseinput.text()[5:].strip()) if self.nextclickCnt == 1: self.book_choose = choose # 选0的话这里要爬虫返回书本数量数据 self.book_num = len(self.book_choose) if self.book_num > 1: self.log.info('book_num > 1') self.textBrowser.append(self.warning_(f'警告！！多选书本时不要随意使用 retry<br>')) self.chooseinput.clear() # choose逻辑 交由crawl, next,retry3个btn的schedule控制 self.params_send({'choose': choose}) self.log.debug(f'send choose: {choose} success') self.retrybtn.setEnabled(True) if self.next_btn.text()!='搜索': _next() else: start_and_search() self.nextclickCnt += 1 self.searchinput.setEnabled(False) self.chooseinput.setFocusPolicy(Qt.StrongFocus) self.step_recv() # 封装的self.step_Q处理方法 self.log.debug(f"===--→ next_schedule end (now step: {self.step})\n") 后台线程后台爬虫进程创建方法 ，上述UI主线程中Next逻辑的 调用
def crawl_what(index, print_Q, bar, current_Q, step_Q): spider_what = {1: 'comic1, 2: 'comic2', 3: 'comic3'} freeze_support() process = CrawlerProcess(get_project_settings()) process.crawl(spider_what[index], print_Q=print_Q, bar=bar, current_Q=current_Q, step_Q=step_Q) process.start() process.join() process.stop()分离UI主线程与工作线程（项目代码中此处可整合爬虫进程一起）
class WorkThread(QThread): item_count_signal = pyqtSignal(int) print_signal = pyqtSignal(str) finishSignal = pyqtSignal(str) active = True def __init__(self, gui): super(WorkThread, self).__init__() self.gui = gui def run(self): while self.active: self.msleep(8) if not self.gui.print_Q.empty(): self.msleep(8) self.print_signal.emit(str(self.gui.print_Q.get())) if not self.gui.bar.empty(): self.item_count_signal.emit(self.gui.bar.get()) self.msleep(10) if '完成任务' in self.gui.textBrowser.toPlainText(): self.item_count_signal.emit(100) self.msleep(20) break if self.active: from ComicSpider.settings import IMAGES_STORE self.finishSignal.emit(IMAGES_STORE) 辅助工具资源处理工具
PYUIC >>> 将.ui界面文件转换成py文件
pyrcc5 >>> 将编入资源路径后的qrc文件，转换成py文件
工具集 utils.py
def get_info(): with open(f'./setting.txt', 'r', encoding='utf-8') as fp: text = fp.read() sv_path = re.findall(r'<([\s\S]*)>', text)[0] level = re.findall('(DEBUG|INFO|ERROR)', text)[0] # ……………… def cLog(name, level='INFO', **kw) -> Logger: # 同理读取setting.txt， level = re.search('(DEBUG|WARNING|ERROR)', text).group(1) def judge_input(_input: str) -> list: """ "6" return [6] // "1+3+5" return [1,3,5] "4-6" return [4,5,6] // "1+4-6" return [1,4,5,6] """ 三、部署部署实为pyinstaller打包成exe
pyinstaller注意要点：
查阅资料和前人摸路，scrapy的打包需要在主运行文件中导入大量模块，可参考 我的配置
spec的datas中每个值前为项目现位置，后为运行时位置；慎用网上传授的('.', '.')，使用不当会使得git体积飞涨
将debug、console设置为True，方便调试 （ 与上述导入模块调试有所关联
spec参考
# -*- mode: python -*- block_cipher = None a = Analysis(['crawl_go.py'], pathex=['D:\\xxxxxxxxxxxxxxxx\\ComicSpider'], binaries=[], datas=[('D:\python\Lib\site-packages\scrapy\mime.types','scrapy'), ('D:\python\Lib\site-packages\scrapy\VERSION','scrapy'), ('./ComicSpider','ComicSpider'), ('./GUI', 'GUI'), ('./gui.py', '.'), ('./material_ct.py', '.'), ('./utils.py', '.'), ], # -*- hiddenimports=[], hookspath=[], runtime_hooks=[], excludes=[], win_no_prefer_redirects=False, win_private_assemblies=False, cipher=block_cipher, noarchive=False) pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) exe = EXE(pyz, a.scripts, a.binaries, a.zipfiles, a.datas, [],, debug=True, # -*- bootloader_ignore_signals=False, strip=False, upx=True, runtime_tmpdir=None, console=True, icon='exe.ico') # -*-