#!/user/bin/env python
# _*_ coding:utf-8 _*_
#!/user/bin/env python
# _*_ coding:utf-8 _*_
import re,requests
import pymysql

headers={'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6','referer':'http://www.17k.com/'}

conn=pymysql.connect(
    host='localhost',
    port=3306,
    user='root',
    passwd='123456',
    db='xiaoshuo',
    charset='utf8'
)
cursor = conn.cursor()

def getNovelInfoList(url):
    response=requests.get(url,headers=headers);
    response.encoding='utf-8';
    html=response.text;
    #获取小说名字
    reg = '''<dt>
					<a href="(.*?)" target="_blank">(.*?)</a>'''
    NovelNameandUrl = re.findall(reg, html, re.S);

    #获取小说作者
    reg='<a href="http://www.17k.com/zuozhe/.*?" target="_blank">(.*?)</a>'
    zuozhe=re.findall(reg, html,re.S);
    #获取小说类别
    reg='<span>类别：<a href="/lib/book/.*?" target="_blank">(.*?)</a></span>'
    NovelCategory=re.findall(reg, html,re.S);
    #获取小说字数
    reg='<span>字数：<code>(.*?)</code></span>'
    NovelNum=re.findall(reg, html,re.S);
    #小说标签
    #获取小说简介
    reg='</strong>.*?<p>.*?<a href="http://www.17k.com/list/.*?" target="_blank">(.*?)</a>'
    NovelBrief=re.findall(reg, html,re.S);
    #获取小说图片
    reg='<img src="(http://img.17k.com/images.*?)" alt=".*?">'
    NovelImg=re.findall(reg, html,re.S);
    return NovelNameandUrl,zuozhe,NovelCategory,NovelNum,NovelBrief,NovelImg

#获取章节列表
def getChapterList(url):
    response=requests.get(url,headers=headers);
    response.encoding='utf-8';
    result=response.text;
    reg=r'<a target="_blank" href="(/chapter/.*?)" title="字数.*?">'
    ChapterUrlList=re.findall(reg,result,re.S);
    reg = r'<span class="ellipsis">(.*?)</span>'
    ChapterNameList = re.findall(reg, result, re.S);
    ChapterInfo={}
    for i in range(len(ChapterNameList)):
        ChapterInfo.setdefault(ChapterNameList[i],'http://www.17k.com'+ChapterUrlList[i]);
    return ChapterInfo

#获取章节内容
def getChapterContent(url):
    response=requests.get(url,headers=headers);
    response.encoding='utf-8';
    result=response.text;
    reg='<div class="p">(.*?)<div class="author-say"></div>'
    ChapterCiontent=re.findall(reg,result,re.S);
    return ChapterCiontent

#修改小说URL
def ModifyNovelUrl(url):
    url=re.sub('book','list',url)
    return url

#获取总页数
def getNovelPages():
    response = requests.get("http://all.17k.com/lib/book/2_0_0_____0_1.html?",headers=headers)
    response.encoding = 'utf-8';
    html = response.text;
    reg = r'共 (.*?) 页 <span>转到第'
    NOvelName = re.findall(reg, html);
    return NOvelName
qqq=1;
for count in range(int(getNovelPages()[0])):
        print('页数' +str(count+1))
        NovelNameandUrl,zuozhe,NovelCategory,NovelNum,NovelBrief,NovelImg =  getNovelInfoList('http://all.17k.com/lib/book/2_0_0_0_0_0_0_1_'+str(count+1)+'.html?')
        #通过小说URL获取小说章节
        a=0;
        b=1;#小说编号
        if NovelNameandUrl:
            for NNU in NovelNameandUrl:
                print(str(b)+'小说名字======'+NNU[1])
                ChapterList=getChapterList(ModifyNovelUrl(NNU[0]))#获取小说章节列表
                try:
                    cursor.execute(
                        "insert into novelinfo (NovelName,zuozhe,NovelCategory,NovelNum,NovelBrief,NovelImg) values('{}','{}','{}','{}','{}','{}')"
                        .format( NNU[1], zuozhe[a], NovelCategory[a], NovelNum[a], NovelBrief[a],NovelImg[a]));
                    conn.commit()
                    lastrowid = cursor.lastrowid  # 存储完一条   返回主键的哪一个int值
                except Exception:
                    print('出现错误      ' + '页数' + str(count +1))
                    continue
                finally:
                    print('总书数')
                #通过小说章节URL获取每一章节内容
                if ChapterList :
                    for CL in ChapterList:
                        if getChapterContent(ChapterList[CL]):
                            ChapterContent=getChapterContent(ChapterList[CL])[0]#获取章节内容

                            try:
                                #存数据库
                                cursor.execute(
                                    "insert into chaptercontent (novelId,chapterName,chaptercntent) values('{}','{}','{}')"
                                        .format(lastrowid, CL, ChapterContent));
                                conn.commit()
                                print('第'+str(a+1)+'本书' + '   ' + NNU[1] + '  ' + CL )

                            except Exception:
                                print('出现错误      '+'页数' +str(count+1))
                                continue
                            finally:
                                qqq = qqq + 1;
                        else:
                            pass
                a=a+1;
                b=b+1;
                if a>=len(NovelNameandUrl):
                    a==0;







#注意
#取出来的都是列表 ，所以取数据  注意是第几条    不能存入数据库的是列表   元祖啥的

#不一定每次都能取出数据来，所以要加入判断，是否为空


#模拟浏览器访问网页
#headers={'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6','referer':'http://www.17k.com/'}
#link为爬虫的入口URL