#!/usr/bin/python -tt # cl-aggregator v1.0.1 - aggregate craigslist rss feeds into a single feed # Copyright (c) 2008 - 2011, John Morrissey # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OUTFILE = '/path/to/rss/output/file' FEEDS = [ 'http://anchorage.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://auburn.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://bham.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://dothan.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://gadsden.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://huntsville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://mobile.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://montgomery.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://shoals.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://tuscaloosa.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fayar.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fortsmith.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://jonesboro.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://littlerock.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://flagstaff.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://mohave.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://phoenix.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://prescott.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sierravista.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://tucson.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://yuma.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://bakersfield.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://chico.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fresno.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://goldcountry.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://humboldt.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://imperial.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://inlandempire.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://mendocino.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://merced.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://modesto.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://monterey.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://orangecounty.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://palmsprings.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://redding.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sacramento.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sandiego.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://slo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://santabarbara.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sfbay.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://stockton.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://ventura.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://visalia.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://yubasutter.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://boulder.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://cosprings.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://denver.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fortcollins.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://pueblo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://rockies.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://westslope.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://hartford.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://newhaven.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://newlondon.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://nwct.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://washingtondc.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://delaware.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://daytona.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://keys.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fortlauderdale.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fortmyers.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://gainesville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://jacksonville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lakeland.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://miami.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://ocala.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://orlando.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://panamacity.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://pensacola.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sarasota.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://spacecoast.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://staugustine.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://tallahassee.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://tampa.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://treasure.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://westpalmbeach.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://athensga.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://atlanta.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://augusta.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://brunswick.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://columbusga.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://macon.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://savannah.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://valdosta.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://honolulu.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://ames.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://cedarrapids.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://desmoines.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://dubuque.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://iowacity.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://quadcities.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://siouxcity.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://waterloo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://boise.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://eastidaho.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://spokane.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://twinfalls.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://bn.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://carbondale.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://chambana.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://chicago.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://decatur.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://peoria.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://rockford.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://springfieldil.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://bloomington.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://evansville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fortwayne.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://indianapolis.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://muncie.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://southbend.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://terrahaute.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://tippecanoe.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lawrence.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://ksu.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://topeka.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://wichita.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://bgky.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lexington.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://louisville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://westky.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://batonrouge.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lafayette.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lakecharles.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://monroe.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://neworleans.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://shreveport.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://boston.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://capecod.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://southcoast.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://westernmass.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://worcester.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://annapolis.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://baltimore.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://easternshore.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://smd.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://westmd.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://maine.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://annarbor.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://centralmich.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://detroit.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://flint.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://grandrapids.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://jxn.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://kalamazoo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lansing.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://muskegon.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://nmi.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://porthuron.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://saginaw.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://up.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://duluth.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://mankato.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://minneapolis.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://rmn.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://stcloud.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://columbiamo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://joplin.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://kansascity.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://semo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://springfield.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://stlouis.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://gulfport.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://hattiesburg.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://jackson.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://northmiss.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://montana.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://asheville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://boone.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://charlotte.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://eastnc.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fayetteville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://greensboro.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://hickory.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://outerbanks.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://raleigh.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://wilmington.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://winstonsalem.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fargo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://nd.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://grandisland.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lincoln.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://omaha.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://nh.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://cnj.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://jerseyshore.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://newjersey.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://southjersey.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://albuquerque.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://farmington.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lascruces.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://roswell.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://santafe.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lasvegas.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://reno.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://albany.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://binghamton.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://buffalo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://catskills.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://chautauqua.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://elmira.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://hudsonvalley.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://ithaca.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://longisland.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://newyork.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://plattsburgh.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://rochester.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://syracuse.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://utica.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://watertown.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://akroncanton.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://athensohio.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://cincinnati.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://cleveland.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://columbus.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://dayton.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://limaohio.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://mansfield.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sandusky.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://toledo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://youngstown.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lawton.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://oklahomacity.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://stillwater.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://tulsa.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://bend.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://corvallis.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://eastoregon.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://eugene.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://medford.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://oregoncoast.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://portland.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://roseburg.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://salem.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://altoona.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://erie.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://harrisburg.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lancaster.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://allentown.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://pennstate.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://philadelphia.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://pittsburgh.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://poconos.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://reading.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://scranton.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://williamsport.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://york.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://puertorico.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://providence.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://charleston.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://columbia.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://florencesc.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://greenville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://hiltonhead.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://myrtlebeach.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sd.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://chattanooga.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://clarksville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://jacksontn.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://knoxville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://memphis.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://nashville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://tricities.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://abilene.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://amarillo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://austin.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://beaumont.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://brownsville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://collegestation.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://corpuschristi.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://dallas.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://elpaso.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://galveston.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://houston.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://killeen.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://laredo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lubbock.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://mcallen.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://odessa.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sanantonio.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sanmarcos.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://texarkana.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://easttexas.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://victoriatx.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://waco.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://wichitafalls.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://logan.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://ogden.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://provo.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://saltlakecity.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://stgeorge.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://blacksburg.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://charlottesville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://danville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://fredericksburg.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://norfolk.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://harrisonburg.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lynchburg.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://richmond.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://roanoke.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://burlington.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://bellingham.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://kpr.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://olympic.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://pullman.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://seattle.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://skagit.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://wenatchee.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://yakima.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://appleton.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://eauclaire.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://greenbay.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://janesville.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://lacrosse.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://madison.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://milwaukee.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://racine.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://sheboygan.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://wausau.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://charlestonwv.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://huntington.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://martinsburg.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://morgantown.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://parkersburg.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://wv.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://wheeling.craigslist.org/search/XXX?query=QUERYformat=rss', 'http://wyoming.craigslist.org/search/XXX?query=QUERYformat=rss', ] from datetime import datetime import sys try: import hashlib md5_constructor = hashlib.md5 except ImportError: import md5 md5_constructor = md5.new import feedparser import PyRSS2Gen seen_hashes = [] dupes = 0 items = [] for feed in FEEDS: d = feedparser.parse(feed) for entry in d['entries']: m = md5_constructor() if not hasattr(entry, 'summary'): continue m.update(entry.summary.encode('utf-8')) hash = str(m.hexdigest()) #print 'entry: %s, %s' % (hash, entry.title.encode('utf-8')) if hash in seen_hashes: dupes += 1 #print 'skipping dupe: %s, %s' % (hash, entry.title.encode('utf-8')) continue seen_hashes.append(hash) items.append(PyRSS2Gen.RSSItem( title=entry.title, link=entry.link, description=entry.description, guid=PyRSS2Gen.Guid(hash), pubDate=entry.updated)) PyRSS2Gen.RSS2( title='craigslist aggregator', link='', description='Craigslist Aggregator', lastBuildDate=datetime.now(), items=items).write_xml(open(OUTFILE, 'w')) #print 'Saw %d unique entries, %d dupes.' % (len(seen_hashes), dupes)