O
onyliu
Unregistered / Unconfirmed
GUEST, unregistred user!
续:写了一个多线程下载网业内容的程序,下载正常,就是程序隔一会CPU占用就到90%(整理了下程序,加了注释,希望大侠们帮我看一下,不要嫌分少啊,就乘45分了,)(45分)<br />procedure TGetMM.Execute;
var
strm:TStringStream;
IdGetMM: TIdHTTP;
url_str,sqlstr,str_temp:string;
count,i : integer;
source,str_url,str_date,str_registdate : string;
ADOQuery: TADOQuery;
begin
try
begin
while 1>0do
begin
for count:=strtoint(FPagebegin
) to StrToInt(FpageEnd)do
//循环翻页
begin
try
if IdGetMM <> nil then
IdGetMM.Free;
////////////////////////1
IdGetMM :=TIdHTTP.Create(nil);
if ADOQuery <> nil then
ADOQuery.Free;
ADOQuery:= TADOQuery.Create(nil);
sleep(10);
ADOQuery.ConnectionString :=ConnStr;
if FUrlInfo <> nil then
FUrlInfo.Free;
FUrlInfo := TStringList.Create;
if FRegistdate <> nil then
FRegistdate.Free;
FRegistdate := TStringList.Create;
sleep(10);
url_str := FUrl + inttostr(count);
//--1创建控件
try
sleep(10);
/////////////////////////////2
strm :=TStringStream.Create('');
IdGetMM.Get(url_str,strm);
source := strm.DataString;
//---2下载网页文档
finally
strm.Free;
end;
sleep(5);
/////////////////////////////////3
if Fbegin
key <> '' then
source := FormTCutString.GetstrFromHTML(Fbegin
key,FEndkey,source);
if FDelkey1 <> '' then
source := FormTCutString.StrReplace(FDelkey1,'',FDelkey2,'',source);
source := FormTCutString.StrReplace('"','','''','',source);
str_url := FormTCutString.StrReplace('<a href=',FMainKey,'<a href=',FMainKey,FSplit,'汗',source);
if Fdatekey1 <> '' then
source := FormTCutString.StrReplace(Fdatekey1,Formatdatetime('YYYY',Now)+'-',Fdatekey2,'-',source);
str_temp := source;
str_date := FormTCutString.ParseDate(str_temp,1);
str_date := str_date+FormTCutString.ParseDate(str_temp,2);
FormTCutString.SeparateTerms(str_date,',',FRegistdate);
str_url := FormTCutString.Parseurl(str_url,FSubKey);
if str_url <>'' then
//----------------3整理分析出URL
begin
FormTCutString.SeparateTerms(str_url,',',FUrlInfo);
for i := 0 to FUrlInfo.Count-1do
begin
if pos('www.xxx.com',FUrlInfo.Strings)= 0 then
begin
sleep(10);
if CheckUrl(Formatdatetime('YYYY-MM-DD',Now),FUrlInfo.Strings,ADOQuery) =0 then
//判断URL是否重复
begin
try /////////////////4
sleep(10);
strm :=TStringStream.Create('');
sleep(10);
url_str := FUrlInfo.Strings;
IdGetMM.Get(url_str,strm);
source := strm.DataString;
//------4下载
finally
strm.Free;
end;
///////////////////5
if Trim(FSubbegin
key) <> '' then
source := FormTCutString.GetstrFromHTML(FSubbegin
key,FSubEndkey,source);
sleep(10);
source := Clear(source);
str_registdate := '';
if FRegistdate.Count > 0 then
str_registdate := FRegistdate.Strings;
if str_registdate = '' then
begin
if (pos(Formatdatetime('YYYY-MM-DD',Now),source)>0) or (pos(Formatdatetime('YYYY-M-DD',Now),source)>0) or (pos(Formatdatetime('MM-DD',Now),source)>0) or (pos(Formatdatetime('M-DD',Now),source)>0) then
str_registdate := Formatdatetime('YYYY-MM-DD',Now);
end;
if (str_registdate = Formatdatetime('YYYY-MM-DD',Now)) then
begin
if (CheckAreaTel(str_registdate,Get(source,TelephoneList),Get(source,AreaList),ADOQuery) =0) and (Get(source,TelephoneList) <>'') then
begin
if (CheckBlackTel(Get(source,TelephoneList),ADOQuery) =0) then
begin
sqlstr :='insert into rentinfo(registdate,Buildtype,Structure,Tracffic,Contact,SellType,base,equipment,mright,fitment,other,url,urltype,siteid,cityid,district,area,provider,telephone,floor,size,room,buildyear,towards,price) values(';
sqlstr :=sqlstr +''''+str_registdate+''''+','+''''+Get(source,BuildtypeList)+''''+','+''''+Get(source,StructureList)+''''+','+''''+Get(source,TracfficList)+''''+','+''''+copy(Get(source,ContactList),0,6)+''''+','+''''+Get(source,SellTypeList)+''''+',';
sqlstr :=sqlstr +''''+Get(source,BaseList)+''''+','+''''+Get(source,EquipmentList)+''''+','+''''+Get(source,MrightList)+''''+','+''''+Get(source,FitmentList)+''''+','+''''+Get(source,OtherList)+''''+',';
sqlstr :=sqlstr +''''+FUrlInfo.Strings+''''+','+FUrltype+','+FSiteID+','+FCityID+','+''''+Get(source,DistrictList)+''''+','+''''+Get(source,AreaList)+''''+','+'''个人'''+','+''''+Get(source,TelephoneList)+''''+','+''''+Get(source,FloorList)+''''+','+''''+Get(source,SizeList)+''''+','+''''+Get(source,RoomList)+''''+','+''''+Get(source,YearList)+''''+','+''''+Get(source,TowardsList)+''''+','+''''+Get(source,PriceList)+''''+')';
ExecSql(sqlstr,ADOQuery);
end;
end;
end;
end;
end;
end;
end;
except
end;
end;
//---------------------------这段就是整理分析文档并存入数据库(插入前做了几次判断(对数据库查询))
end;
sleep(StrTOInt(FSleepTime));
end;
except
end;
end;
var
strm:TStringStream;
IdGetMM: TIdHTTP;
url_str,sqlstr,str_temp:string;
count,i : integer;
source,str_url,str_date,str_registdate : string;
ADOQuery: TADOQuery;
begin
try
begin
while 1>0do
begin
for count:=strtoint(FPagebegin
) to StrToInt(FpageEnd)do
//循环翻页
begin
try
if IdGetMM <> nil then
IdGetMM.Free;
////////////////////////1
IdGetMM :=TIdHTTP.Create(nil);
if ADOQuery <> nil then
ADOQuery.Free;
ADOQuery:= TADOQuery.Create(nil);
sleep(10);
ADOQuery.ConnectionString :=ConnStr;
if FUrlInfo <> nil then
FUrlInfo.Free;
FUrlInfo := TStringList.Create;
if FRegistdate <> nil then
FRegistdate.Free;
FRegistdate := TStringList.Create;
sleep(10);
url_str := FUrl + inttostr(count);
//--1创建控件
try
sleep(10);
/////////////////////////////2
strm :=TStringStream.Create('');
IdGetMM.Get(url_str,strm);
source := strm.DataString;
//---2下载网页文档
finally
strm.Free;
end;
sleep(5);
/////////////////////////////////3
if Fbegin
key <> '' then
source := FormTCutString.GetstrFromHTML(Fbegin
key,FEndkey,source);
if FDelkey1 <> '' then
source := FormTCutString.StrReplace(FDelkey1,'',FDelkey2,'',source);
source := FormTCutString.StrReplace('"','','''','',source);
str_url := FormTCutString.StrReplace('<a href=',FMainKey,'<a href=',FMainKey,FSplit,'汗',source);
if Fdatekey1 <> '' then
source := FormTCutString.StrReplace(Fdatekey1,Formatdatetime('YYYY',Now)+'-',Fdatekey2,'-',source);
str_temp := source;
str_date := FormTCutString.ParseDate(str_temp,1);
str_date := str_date+FormTCutString.ParseDate(str_temp,2);
FormTCutString.SeparateTerms(str_date,',',FRegistdate);
str_url := FormTCutString.Parseurl(str_url,FSubKey);
if str_url <>'' then
//----------------3整理分析出URL
begin
FormTCutString.SeparateTerms(str_url,',',FUrlInfo);
for i := 0 to FUrlInfo.Count-1do
begin
if pos('www.xxx.com',FUrlInfo.Strings)= 0 then
begin
sleep(10);
if CheckUrl(Formatdatetime('YYYY-MM-DD',Now),FUrlInfo.Strings,ADOQuery) =0 then
//判断URL是否重复
begin
try /////////////////4
sleep(10);
strm :=TStringStream.Create('');
sleep(10);
url_str := FUrlInfo.Strings;
IdGetMM.Get(url_str,strm);
source := strm.DataString;
//------4下载
finally
strm.Free;
end;
///////////////////5
if Trim(FSubbegin
key) <> '' then
source := FormTCutString.GetstrFromHTML(FSubbegin
key,FSubEndkey,source);
sleep(10);
source := Clear(source);
str_registdate := '';
if FRegistdate.Count > 0 then
str_registdate := FRegistdate.Strings;
if str_registdate = '' then
begin
if (pos(Formatdatetime('YYYY-MM-DD',Now),source)>0) or (pos(Formatdatetime('YYYY-M-DD',Now),source)>0) or (pos(Formatdatetime('MM-DD',Now),source)>0) or (pos(Formatdatetime('M-DD',Now),source)>0) then
str_registdate := Formatdatetime('YYYY-MM-DD',Now);
end;
if (str_registdate = Formatdatetime('YYYY-MM-DD',Now)) then
begin
if (CheckAreaTel(str_registdate,Get(source,TelephoneList),Get(source,AreaList),ADOQuery) =0) and (Get(source,TelephoneList) <>'') then
begin
if (CheckBlackTel(Get(source,TelephoneList),ADOQuery) =0) then
begin
sqlstr :='insert into rentinfo(registdate,Buildtype,Structure,Tracffic,Contact,SellType,base,equipment,mright,fitment,other,url,urltype,siteid,cityid,district,area,provider,telephone,floor,size,room,buildyear,towards,price) values(';
sqlstr :=sqlstr +''''+str_registdate+''''+','+''''+Get(source,BuildtypeList)+''''+','+''''+Get(source,StructureList)+''''+','+''''+Get(source,TracfficList)+''''+','+''''+copy(Get(source,ContactList),0,6)+''''+','+''''+Get(source,SellTypeList)+''''+',';
sqlstr :=sqlstr +''''+Get(source,BaseList)+''''+','+''''+Get(source,EquipmentList)+''''+','+''''+Get(source,MrightList)+''''+','+''''+Get(source,FitmentList)+''''+','+''''+Get(source,OtherList)+''''+',';
sqlstr :=sqlstr +''''+FUrlInfo.Strings+''''+','+FUrltype+','+FSiteID+','+FCityID+','+''''+Get(source,DistrictList)+''''+','+''''+Get(source,AreaList)+''''+','+'''个人'''+','+''''+Get(source,TelephoneList)+''''+','+''''+Get(source,FloorList)+''''+','+''''+Get(source,SizeList)+''''+','+''''+Get(source,RoomList)+''''+','+''''+Get(source,YearList)+''''+','+''''+Get(source,TowardsList)+''''+','+''''+Get(source,PriceList)+''''+')';
ExecSql(sqlstr,ADOQuery);
end;
end;
end;
end;
end;
end;
end;
except
end;
end;
//---------------------------这段就是整理分析文档并存入数据库(插入前做了几次判断(对数据库查询))
end;
sleep(StrTOInt(FSleepTime));
end;
except
end;
end;