p15679402
/
bigdata
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

.

This commit is contained in:
guange 2019-01-14 16:40:25 +08:00
parent 63c091e6e1
commit 1a2e31cd14
5 changed files with 89 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
chapter1/.idea/workspace.xml
View File

@ -1,14 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="a5fbb387-9969-4874-8a7f-c9dd40d5225d" name="Default Changelist" comment="增加id字段">
<change afterPath="$PROJECT_DIR$/crawler/taobao/spiders/jd.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/crawler/taobao/items.py" beforeDir="false" afterPath="$PROJECT_DIR$/crawler/taobao/items.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/crawler/taobao/pipelines.py" beforeDir="false" afterPath="$PROJECT_DIR$/crawler/taobao/pipelines.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/crawler/taobao/settings.py" beforeDir="false" afterPath="$PROJECT_DIR$/crawler/taobao/settings.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/crawler/taobao/spiders/comment.py" beforeDir="false" afterPath="$PROJECT_DIR$/crawler/taobao/spiders/comment.py" afterDir="false" />
</list>
<list default="true" id="a5fbb387-9969-4874-8a7f-c9dd40d5225d" name="Default Changelist" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -100,8 +93,8 @@
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/crawler/taobao/spiders/jd.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="136">
<caret line="60" column="21" lean-forward="true" selection-start-line="60" selection-start-column="21" selection-end-line="60" selection-end-column="21" />
<state relative-caret-position="241">
<caret line="67" column="24" lean-forward="true" selection-start-line="67" selection-start-column="24" selection-end-line="67" selection-end-column="24" />
<folding>
<element signature="e#14#27#0" expanded="true" />
</folding>
@ -322,7 +315,14 @@
<option name="project" value="LOCAL" />
<updated>1546938314632</updated>
</task>
<option name="localTasksCounter" value="3" />
<task id="LOCAL-00003" summary="加入京东抓取">
<created>1547454506424</created>
<option name="number" value="00003" />
<option name="presentableId" value="LOCAL-00003" />
<option name="project" value="LOCAL" />
<updated>1547454506424</updated>
</task>
<option name="localTasksCounter" value="4" />
<servers />
</component>
<component name="TodoView">
@ -336,6 +336,7 @@
</component>
<component name="ToolWindowManager">
<frame x="0" y="23" width="1280" height="777" extended-state="0" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.25767368" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
@ -360,7 +361,8 @@
<component name="VcsManagerConfiguration">
<MESSAGE value="数据抓取" />
<MESSAGE value="增加id字段" />
<option name="LAST_COMMIT_MESSAGE" value="增加id字段" />
<MESSAGE value="加入京东抓取" />
<option name="LAST_COMMIT_MESSAGE" value="加入京东抓取" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/crawler/scrapy.cfg">
@ -585,8 +587,8 @@
</entry>
<entry file="file://$PROJECT_DIR$/crawler/taobao/spiders/jd.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="136">
<caret line="60" column="21" lean-forward="true" selection-start-line="60" selection-start-column="21" selection-end-line="60" selection-end-column="21" />
<state relative-caret-position="241">
<caret line="67" column="24" lean-forward="true" selection-start-line="67" selection-start-column="24" selection-end-line="67" selection-end-column="24" />
<folding>
<element signature="e#14#27#0" expanded="true" />
</folding>

BIN
chapter1/crawler/taobao/__pycache__/items.cpython-37.pyc
View File

Binary file not shown.

BIN
chapter1/crawler/taobao/__pycache__/pipelines.cpython-37.pyc
View File

Binary file not shown.

BIN
chapter1/crawler/taobao/__pycache__/settings.cpython-37.pyc
View File

Binary file not shown.

73
chapter1/requirements.txt Normal file
View File

@ -0,0 +1,73 @@
appnope==0.1.0
asn1crypto==0.24.0
attrs==18.2.0
Automat==0.7.0
backcall==0.1.0
bleach==3.1.0
cffi==1.11.5
constantly==15.1.0
cryptography==2.4.2
cssselect==1.0.3
decorator==4.3.0
defusedxml==0.5.0
entrypoints==0.3
hyperlink==18.0.0
idna==2.8
incremental==17.5.0
ipykernel==5.1.0
ipython==7.2.0
ipython-genutils==0.2.0
ipywidgets==7.4.2
jedi==0.13.2
Jinja2==2.10
jsonschema==2.6.0
jupyter==1.0.0
jupyter-client==5.2.4
jupyter-console==6.0.0
jupyter-core==4.4.0
lxml==4.3.0
MarkupSafe==1.1.0
mistune==0.8.4
nbconvert==5.4.0
nbformat==4.4.0
notebook==5.7.4
numpy==1.15.4
pandas==0.23.4
pandocfilters==1.4.2
parsel==1.5.1
parso==0.3.1
pexpect==4.6.0
pickleshare==0.7.5
Pillow==5.4.1
prometheus-client==0.5.0
prompt-toolkit==2.0.7
ptyprocess==0.6.0
pyasn1==0.4.5
pyasn1-modules==0.2.3
pycparser==2.19
PyDispatcher==2.0.5
Pygments==2.3.1
PyHamcrest==1.9.0
pyOpenSSL==18.0.0
python-dateutil==2.7.5
pytz==2018.9
pyzmq==17.1.2
qtconsole==4.4.3
queuelib==1.5.0
Scrapy==1.5.1
scrapy-splash==0.7.2
selenium==3.141.0
Send2Trash==1.5.0
service-identity==18.1.0
six==1.12.0
terminado==0.8.1
testpath==0.4.2
tornado==5.1.1
traitlets==4.3.2
Twisted==18.9.0
urllib3==1.24.1
w3lib==1.19.0
wcwidth==0.1.7
webencodings==0.5.1
widgetsnbextension==3.4.2
zope.interface==4.6.0