Tons of fixes

have downloaded a complete history of the sp500 since 01/01/2000

6 files changed, 189 insertions, 170 deletions

diff --git a/bandit.R b/bandit.R
index 1a8dd1b..8e2fe70 100644
--- a/bandit.R
+++ b/bandit.R
@@ -1,10 +1,4 @@
-#fix bug in adjRatios
-source("patch-adjRatios.R")
-assignInNamespace("adjRatios",adjRatios,ns="TTR")
-library(TTR)
-rm(adjRatios)
-
-sp500<- read.table("sp500 tickers 10-04.csv",sep="\t",header=T,quote="",colClasses="character",strip.white=T)
+sp500<- read.table("sp500 tickers 10-17.csv",sep="\t",header=T,quote="",colClasses="character",strip.white=T)
 colnames(sp500) <- c("name","ticker")
 #replace / by -
 sp500$ticker <- sub('/', '-', sp500$ticker)
@@ -14,6 +8,7 @@ tickerlist <- list()
 for(i in 1:length(sp500$ticker)){
   ticker <- sp500[i,"ticker"]
   tickerlist[[ticker]] <- getYahooData(ticker,"20000101")
+  cat("ticker",i,ticker,"\n")
 }
 
 #compute matrix of returns
diff --git a/bloomberg-data.R b/bloomberg-data.R
index 229254d..2c37c20 100644
--- a/bloomberg-data.R
+++ b/bloomberg-data.R
@@ -1,51 +1,35 @@
 require(RBloomberg)
-source("getBloombergData.R")
+source("utils.R")
 conn <- blpConnect(jvm.params = "-Xmx1024m")
+
+#get current tickers
 sp500.tickers <- as.character(bds(conn,"SPX Index","INDX_MEMBERS")[,1])
 #remove exchange information
-for(i in 1:length(sp500.tickers)){
-  ticker <- sp500.tickers[i]
-  sp500.tickers[i] <- strsplit(ticker," ")[[1]][1]
+sp500.tickers <- gsub(" [A-Z]*","", sp500.tickers)
+
+add <- read.table("sp500 add.csv", sep=",", fill=T, header=T,
+                  colClasses="character", quote="")
+add$date <- as.Date(add$date,format="%m/%d/%Y")
+sp500.extended <- union(sp500.tickers,add$ticker.del)
+sp500.extended <- sp500.extended[-match(c("","WHO"),sp500.extended)]
+extra.tickers <- setdiff(sp500.extended,sp500.tickers)
+company.names <- c()
+for(ticker in sp500.extended){
+  company.names <- c(company.names,
+                     bdp(conn, paste(ticker, "Equity"), "NAME")$NAME)
 }
+sp500.extended <- data.frame(ticker=sp500.extended,name=company.names)
+write.table(file="sp500 extended.csv", sp500.extended, col.names=T,
+            row.names=F, sep=",")
 
+#retrieve data from bloomberg
 start.date <- as.Date("2000-01-01")
-list.sp500 <- list()
-for(i in 1:500){
-  ticker <- sp500.tickers[i]
-  #list.sp500.new[[ticker]] <- getBloombergData(conn,ticker,start.date)
-  list.sp500[[ticker]] <- bdh(conn,paste(ticker,"Equity"),c("PX_OPEN","PX_HIGH","PX_LOW","PX_LAST","VOLUME"),start.date,dates.as.row.names=F)
+stock.data <- list()
+for(i in 1:NROW(sp500.extended)){
+  ticker <- as.character(sp500.extended$ticker[i])
+  stock.data[[ticker]] <- getBloombergData(conn,ticker,start.date)
   cat(paste("ticker", i, ": ",ticker),"\n")
 }
-test <- c()
-for(i in 1:158){
-  test <- c(test,sum(abs(as.numeric(list.sp500.new[[i]][1,1:5])-as.numeric(list.sp500[[sp500.tickers[i]]][1,2:6]))))
-}
 
-add <- read.table("sp500 add.csv",sep=",",fill=T,header=T,colClasses="character",quote="")
-add$date <- as.Date(add$date,format="%m/%d/%Y")
 
-memb <- function(index,add,date){
-  #return the list of index constituents
-  toreverse <- add[add$date>=date,]
-  current.index <- index
-  for(i in 1:nrow(toreverse)){
-    if(toreverse$ticker.add[i]!=""){
-      current.index <- current.index[-match(toreverse$ticker.add[i],current.index)]
-    }
-    if(toreverse$ticker.del[i]!=""){
-      current.index <- sort(c(current.index,toreverse$ticker.del[i]))
-    }
-  }
-  current.index
-}
-
-sp500.extended <- union(sp500.tickers,add$ticker.del)
-sp500.extended <- sp500.extended[-match("",sp500.extended)]
-extra.tickers <- setdiff(sp500.extended,sp500.tickers)
-
-list.extra <- list()
-for(i in 1:length(extra.tickers)){
-  ticker <- extra.tickers[i]
-  list.extra[[ticker]] <- bdh(conn,paste(ticker,"Equity"),c("PX_OPEN","PX_HIGH","PX_LOW","PX_LAST"),as.Date("2000-01-01"),dates.as.row.names=F)
-}
 
diff --git a/patch-adjRatios.R b/patch-adjRatios.R
index a96f541..e7beeab 100644
--- a/patch-adjRatios.R
+++ b/patch-adjRatios.R
@@ -42,8 +42,12 @@ adjRatios <- function (splits, dividends, close)
     obj <- merge.xts(close, splits, dividends)
     # for a non NA dividend value, close from the previous day must be non NA
     #otherwise we can't compute the dividend ratio
-    if(any(is.na(close[-length(close)][!is.na(dividends)[-1]]))){
-      stop("can't compute dividend ratio")
+    if(any(is.na(obj[-NROW(obj),1][!is.na(obj[,3])[-1]]))){
+      for(i in which(!is.na(obj[,3]))){
+        if( is.na(obj[i-1,1]) ){
+          obj[i-1,1] <- last(na.omit(obj[1:(i-1),1]))
+        }
+      }
     }
     # if close is NA, then we can't compute the dividend ratio anway
     #if (!isTRUE(is.na(close))) {
diff --git a/sp500 add.csv b/sp500 add.csv
index 49507a1..2199b98 100644
--- a/sp500 add.csv
+++ b/sp500 add.csv
@@ -18,7 +18,7 @@ date,ticker.add,name.add,ticker.del,name.del
 12/20/2010,NFX,NEWFIELD EXPLORATION CO,EK,EASTMAN KODAK CO
 11/17/2010,IR,INGERSOLL-RAND PLC,PTV,PACTIV CORPORATION
 8/27/2010,TYC,TYCO INTL,SII,SMITH INTL INC
-7/15/2010,ACE,ACE LIMITED,MIL,MILLIPORE CORP
+7/15/2010,ACE,ACE LIMITED,0202445Q,MILLIPORE CORP
 7/1/2010,QEP,QEP RESOURCES INC,STR,QUESTAR CORP
 6/28/2010,KMX,CARMAX INC,XTO,XTO ENERGY INC
 4/30/2010,CERN,CERNER CORP,BJS,BJ SERVICES CO
@@ -26,11 +26,12 @@ date,ticker.add,name.add,ticker.del,name.del
 3/1/2010,DISCA,DISCOVERY COMMUNICATIONS,PBG,PEPSI BOTTLING GROUP INC
 3/1/2010,HP,HELMERICH & PAYNE INC,RX,IMS HEALTH INC
 2/16/2010,BRK/B,BERKSHIRE HATHAWAY B,BNI,BURLINGTON NORTHERN SANTE FE
-2/8/2010,URBN,URBAN OUTFITTERS,ACS,AFFILIATED COMPUTER SERVICES
+2/8/2010,URBN,URBAN OUTFITTERS,8277831Q,AFFILIATED COMPUTER SERVICES
 1/29/2010,NRG,NRG ENERGY,JAVA,SUN MICROSYSTEMS INC
+1/29/2010,MJN,,MJN-W
 12/23/2009,ROP,ROPER INDUSTRIES INC,ESV,ENSCO INTL INC
 12/21/2009,CLF,CLIFFS NATURAL RESOURCES,KBH,KB HOME
-12/21/2009,MJN,MEAD JOHNSON NUTRITION CO,MBI,MBIA INC
+12/21/2009,MJN-W,MEAD JOHNSON NUTRITION CO,MBI,MBIA INC
 12/21/2009,ROST,ROSS STORES INC,DYN,DYNEGY INC A
 12/21/2009,SAI,SAIC INC,CVG,CONVERGYS CORP
 12/21/2009,V,VISA INC,CIEN,CIENA CORP
@@ -40,14 +41,14 @@ date,ticker.add,name.add,ticker.del,name.del
 9/9/2009,ARG,AIRGAS INC,CBE,COOPER INDUSTRIES LTD
 9/1/2009,CFN,CAREFUSION CORP,MTW,MANITOWOC CO INC
 8/19/2009,FMC,FMC CORP,CTX,CENTEX CORP
-7/25/2009,RHT,RED HAT INC,CIT,CIT GROUP INC
+7/25/2009,RHT,RED HAT INC,CITGQ,CIT GROUP INC
 7/1/2009,PWR,QUANTA SERVICES INC,IR,INGERSOLL-RAND COMPANY LTD
 7/1/2009,WDC,WESTERN DIGITAL CORP.,EQ,EMBARQ CORP.
 6/30/2009,PCS,METROPCS COMMUNICATIONS
 6/26/2009,,,TEL,TYCO ELECTRONICS LTD
 6/9/2009,DV,DEVRY INC
 6/5/2009,FTI,FMC TECHNOLOGIES INC.,COV,COVIDIEN LTD.
-6/3/2009,,,GM,GENERAL MOTORS CORP
+6/3/2009,,,MTLQQ,MOTORS LIQUIDATION CO
 4/2/2009,DNR,DENBURY RESOURCES INC,ROH,ROHM & HAAS CO
 3/30/2009,TWC,TIME WARNER CABLE INC,DDR,DEVELOPERS DIVERSIFIED REALTY
 3/27/2009,ORLY,O'REILLY AUTOMOTIVE,NE,NOBLE CORP
@@ -56,7 +57,7 @@ date,ticker.add,name.add,ticker.del,name.del
 3/4/2009,VTR,VENTAS INC,JNY,JONES APPAREL GROUP
 2/26/2009,DO,DIAMOND OFFSHORE DRILLING,WFT,WEATHERFORD INTERNATIONAL LTD
 1/30/2009,HCN,HEALTH CARE REIT INC,SOV,SOVEREIGN BANCORP
-1/6/2009,IRM,IRON MOUNTAIN INC,UST,UST INC
+1/6/2009,IRM,IRON MOUNTAIN INC,3517233Q,UST INC
 1/2/2009,FLIR,FLIR SYSTEMS,NCC,NATIONAL CITY CORP
 1/2/2009,OI,OWENS-ILLINOIS,WB,WACHOVIA CORP(NEW)
 1/2/2009,SCG,SCANA CORP,MER,MERRILL LYNCH
@@ -64,8 +65,8 @@ date,ticker.add,name.add,ticker.del,name.del
 12/19/2008,EQT,EQUITABLE RESOURCES,RIG,TRANSOCEAN INC (NEW)
 12/5/2008,RSG,REPUBLIC SERVICES,AW,ALLIED WASTE INDUSTRIES
 12/2/2008,DNB,DUN & BRADSTREET,LIZ,LIZ CLAIBORNE
-11/24/2008,LIFE,LIFE TECHNOLOGIES CORP,ABI,APPLIED BIOSYSTEMS INC
-11/19/2008,SRCL,STERICYCLE INC,BUD,ANHEUSER-BUSCH
+11/24/2008,LIFE,LIFE TECHNOLOGIES CORP,3286728Q,APPLIED BIOSYSTEMS INC
+11/19/2008,SRCL,STERICYCLE INC,3393199Q,ANHEUSER-BUSCH
 11/17/2008,CEPH,CEPHALON INC
 11/14/2008,WYNN,WYNN RESORTS LTD,ASH,ASHLAND INC
 11/14/2008,XRAY,DENTSPLY INTERNATIONAL IN,HPC,HERCULES INC
@@ -77,40 +78,40 @@ date,ticker.add,name.add,ticker.del,name.del
 10/22/2008,NDAQ,NASDAQ OMX GROUP INC,DDS,DILLARD'S INC
 10/7/2008,DPS,DR PEPPER SNAPPLE GROUP I,WWY,WRIGLEY WM JR
 10/2/2008,FLS,FLOWSERVE CORPORATION
-9/30/2008,APH,AMPHENOL CORP,WEN,WENDY'S INTERNATIONAL
-9/30/2008,,,WM,WASHINGTON MUTUAL
+9/30/2008,APH,AMPHENOL CORP,3022239Q,WENDY'S INTERNATIONAL
+9/30/2008,,,WAMUQ,WASHINGTON MUTUAL
 9/24/2008,PXD,PIONEER NATURAL RESOURCES
 9/23/2008,,,SAF,SAFECO CORP
 9/22/2008,HRS,HARRIS CORP
-9/17/2008,,,LEH,LEHMAN BROS
+9/17/2008,,,LEHMQ,LEHMAN BROS
 9/15/2008,CRM,SALESFORCE.COM
 9/15/2008,FAST,FASTENAL CO
-9/11/2008,,,FNM,FANNIE MAE
-9/11/2008,,,FRE,FEDERAL HOME LOAN MTG
-8/27/2008,CF,CF INDUSTRIES HOLDINGS IN,EDS,ELECTRONIC DATA SYSTEMS CORP
+9/11/2008,,,FNMA,FANNIE MAE
+9/11/2008,,,FMCC,FEDERAL HOME LOAN MTG
+8/27/2008,CF,CF INDUSTRIES HOLDINGS IN,3423152Q,ELECTRONIC DATA SYSTEMS CORP
 8/21/2008,IVZ,INVESCO LTD,IACI,IAC/INTERACTIVE CORP
-7/31/2008,DVA,DAVITA INC,CCU,CLEAR CHANNEL COMMUNICATIONS
+7/31/2008,DVA,DAVITA INC,2968900Q,CLEAR CHANNEL COMMUNICATIONS
 7/18/2008,MA,MASTERCARD INC,ACE,ACE LIMITED
 7/1/2008,AKS,AK STEEL HOLDING CORP,CFC,COUNTRYWIDE FINANCIAL CORP
 7/1/2008,SNI,SCRIPPS NETWORK INTERACTI,SSP,SCRIPPS (E.W.) 'A'
 6/23/2008,COG,CABOT OIL & GAS,BC,BRUNSWICK CORP
 6/23/2008,MEE,MASSEY ENERGY COMPANY,OMX,OFFICEMAX INC
-6/11/2008,LO,LORILLARD INC,ABK,AMBAC FINANCIAL GROUP
+6/11/2008,LO,LORILLARD INC,ABKFQ,AMBAC FINANCIAL GROUP
 6/6/2008,SWN,SOUTHWESTERN ENERGY,TT,TRANE INC
-6/2/2008,ISRG,INTUITIVE SURGICAL INC,BSC,BEAR STEARNS COS
+6/2/2008,ISRG,INTUITIVE SURGICAL INC,2942331Q,BEAR STEARNS COS
 3/31/2008,HCP,HCP INC,CBH,COMMERCE BANCORP
-3/31/2008,PM,PHILIP MORRIS INTL,CC,CIRCUIT CITY GROUP
+3/31/2008,PM,PHILIP MORRIS INTL,CCTYQ,CIRCUIT CITY GROUP
 1/29/2008,CAM,CAMERON INTERNATIONAL COR,HET,HARRAH'S ENTERTAINMENT
 1/2/2008,TSS,TOTAL SYSTEM SERVICES,SNV,SYNOVUS FINANCIAL
 12/31/2007,WPO,WASHINGTON POST,TIN,TEMPLE-INLAND
 12/21/2007,RRC,RANGE RESOURCES CORP,TRB,TRIBUNE CO
 12/14/2007,GME,GAMESTOP CORP,DJ,DOW JONES & CO
 11/27/2007,RIG,TRANSOCEAN INC (NEW),RIG,TRANSOCEAN (OLD)
-11/19/2007,AMT,AMERICAN TOWER CORP,AT,ALLTEL CORP
+11/19/2007,AMT,AMERICAN TOWER CORP,3692470Q,ALLTEL CORP
 11/16/2007,MTW,MANITOWOC CO,TEK,TEKTRONIX INC
 11/9/2007,POM,PEPCO HOLDINGS INC,HCR,MANOR CARE INC
 10/29/2007,TIE,TITANIUM METALS,BOL,BAUSCH & LOMB
-10/26/2007,JEC,JACOBS ENGINEERING GROUP,AV,AVAYA INC
+10/26/2007,JEC,JACOBS ENGINEERING GROUP,3414924Q,AVAYA INC
 10/25/2007,NYX,NYSE EURONEXT,HLT,HILTON HOTELS
 10/10/2007,EXPD,EXPEDITORS INT'L,TXU,TXU CORP
 10/8/2007,NBL,NOBLE ENERGY INC,ASN,ARCHSTONE-SMITH TRUST
@@ -119,25 +120,25 @@ date,ticker.add,name.add,ticker.del,name.del
 9/27/2007,TSO,TESORO PETROLEUM CO,MXIM,MAXIM INTEGRATED PROD
 9/26/2007,ICE,INTERCONTINENTALEXCHANGE
 9/25/2007,,,FDC,FIRST DATA CORP
-9/7/2007,MCHP,MICROCHIP TECHNOLOGY,CBSS,COMPASS BANCSHARES
+9/7/2007,MCHP,MICROCHIP TECHNOLOGY,8206908Q,COMPASS BANCSHARES
 8/27/2007,LUK,LEUCADIA NATIONAL CORP,KSE,KEYSPAN ENERGY
 7/12/2007,AKAM,AKAMAI TECHNOLOGIES INC,BMET,BIOMET INC
-7/9/2007,ACAS,AMERICAN CAPITAL STRATEGI,DG,DOLLAR GENERAL
+7/9/2007,ACAS,AMERICAN CAPITAL STRATEGI,3403545Q,DOLLAR GENERAL
 7/2/2007,COV,COVIDIEN LTD,SANM,SANMINA-SCI CORP
 7/2/2007,DFS,DISCOVER FINANCIAL SERVIC,ADCT,ADC TELECOMMUNICATIONS
 7/2/2007,GGP,GENL GROWTH PROPERTIES,MEL,MELLON BANK CORP
 7/2/2007,TEL,TYCO ELECTRONICS LTD,PMCS,PMC-SIERRA INC
 7/2/2007,TYC,TYCO INTERNATIONAL-W/I(NE,TYC,TYCO INTERNATIONAL (OLD)
 6/1/2007,PCP,PRECISION CASTPARTS,MEDI,MEDIMMUNE INC
-5/31/2007,WFR,MEMC ELECTRONIC MATERIALS,KMI,KINDER MORGAN INC
-4/10/2007,AIZ,ASSURANT INC,H,REALOGY CORPORATION
+5/31/2007,WFR,MEMC ELECTRONIC MATERIALS,4621824Q,KINDER MORGAN INC
+4/10/2007,AIZ,ASSURANT INC,3362726Q,REALOGY CORPORATION
 4/2/2007,KFT,KRAFT FOODS INC - A,TSG,SABRE HOLDING CORP
 3/29/2007,ANF,ABERCROMBIE & FITCH CO,UVN,UNIVISION COMMUNICATIONS
 3/23/2007,DDR,DEVELOPERS DIVERSIFIED RL,CMX,CAREMARK RX
 3/20/2007,HST,HOST HOTELS & RESORTS,PD,PHELPS DODGE
 3/2/2007,CHRW,C.H. ROBINSON WORLDWIDE,HMA,HEALTH MANAGEMENT ASSOC
 2/22/2007,TEG,INTEGRYS ENERGY GROUP INC,PGL,PEOPLES ENERGY
-2/15/2007,HCBK,HUDSON CITY BANCORP,APCC,AMERICAN POWER CONVERSION
+2/15/2007,HCBK,HUDSON CITY BANCORP,2307532Q,AMERICAN POWER CONVERSION
 2/12/2007,VAR,VARIAN MEDICAL SYSTEMS,EOP,EQUITY OFFICE PROPERTIES
 2/2/2007,RL,POLO RALPH LAUREN CORP,CMVT,Converse
 1/10/2007,AVB,AVALON BAY COMMUNITIES,2987536Q,SYMBOL TECHNOLOGIES
@@ -147,193 +148,195 @@ date,ticker.add,name.add,ticker.del,name.del
 12/4/2006,DTV,DIRECTV GROUP INC,FSL/B,FREESCALE SEMICONDUCTOR INC
 12/1/2006,IACI,IAC/INTERACTIVE CORP,LU,LUCENT TECHNOLOGIES
 12/1/2006,STR,QUESTAR CORP,NFB,NORTH FORK BANCORPORATION
-11/20/2006,BTU,PEABODY ENERGY CORP,HCA,HCA INC
+11/20/2006,BTU,PEABODY ENERGY CORP,3605818Q,HCA INC
 11/17/2006,CTSH,COGNIZANT TECHNOLOGY SOLU,ACV,ALBERTO-CULVER
 11/10/2006,CBG,CB RICHARD ELLIS GROUP,FSH,FISHER SCIENTIFIC INTL INC
-11/6/2006,CELG,Celgene Corp.,ASO,AmSouth Bancorp
+11/10/2006,FIS,FIDELITY NATIONAL INFORMATIO,LPX,LOUISIANA-PACIFIC CORP
+11/6/2006,CELG,Celgene Corp.,3133754Q,AmSouth Bancorp
 10/2/2006,SII,Smith International,GDW,Golden West Financial
 10/2/2006,WU,Western Union Co,ANDW,Andrew Corp
 8/11/2006,CME,Chicago Mercantile Exchan,KMG,Kerr-McGee Corp.
-8/1/2006,H,Realogy Corp.,GTW,Gateway Inc.
-8/1/2006,WYN,Wyndham Worldwide Corp.,CD,Cendant Corp.
+8/1/2006,3362726Q,Realogy Corp.,GTW,Gateway Inc.
+8/1/2006,WYN,Wyndham Worldwide Corp.,CAR,AVIS BUDGET GROUP INC
 7/18/2006,WIN,Windstream Corp.,CTB,Cooper Tire & Rubber Co.
 6/28/2006,CNX,CONSOL Energy Inc.,KRI,Knight-Ridder Inc.
-6/6/2006,CBH,Commerce Bancorp,EC,Engelhard Corp.
+6/6/2006,CBH,Commerce Bancorp,2999130Q,Engelhard Corp.
 6/2/2006,JNPR,Juniper Neworks Inc.,ABS,Albertson's Inc.
 5/18/2006,EQ,Embarq Corp.,AMCC,Applied Micro Circuits Corp.
 4/24/2006,LM,Legg Mason,GDT,Guidant Co.
 4/20/2006,SNDK,SanDisk Corp.,CHIR,Chiron Corp.
 4/3/2006,BXP,Boston Properties,CIN,Cinergy Corp.
-4/3/2006,DF,Dean Foods,MYG,Maytag Corp.
-4/3/2006,GOOG,Google Inc.,BR,Burlington Resources Inc.
+4/3/2006,DF,Dean Foods,3080867Q,Maytag Corp.
+4/3/2006,GOOG,Google Inc.,929530Q,Burlington Resources Inc.
 4/3/2006,KIM,Kimco Realty Corp.,JP,Jefferson-PilotCorp.
-3/3/2006,CHK,Chesapeake Energy Corp.,DCN,Dana Corp.
+3/3/2006,CHK,Chesapeake Energy Corp.,DCNAQ,Dana Corp.
 2/27/2006,BRL,Barr Pharmaceuticals Inc.,3026360Q,Scientific-Atlanta
 2/1/2006,HAR,Harman Int'l,SEBL,Siebel Systems Inc.
 2/1/2006,VRSN,VeriSign Inc.,RBK,Reebok Int'l
 1/5/2006,EL,Estee Lauder
 1/4/2006,,,MERQ,Mercury Interactive
-1/3/2006,VIA/B,Viacom Inc. (New),VC,Visteon Corp.
+1/3/2006,VIA/B,Viacom Inc. (New),VSTNQ,Visteon Corp.
 1/3/2006,WFM,Whole Foods Market,KRB,MBNA Corp.
 12/20/2005,SSP,EW Scripps Co,GP,Georgia-Pacific Group
-12/2/2005,GNW,Genworth Financial Inc.,CPN,Calpine Corp.
+12/2/2005,GNW,Genworth Financial Inc.,CPNLQ,Calpine Corp.
 11/21/2005,AMZN,Amazon.com,653707Q,AT&T Corp.
-10/11/2005,PDCO,Patterson Companies Inc.,DPH,Delphi Corp.
+10/11/2005,PDCO,Patterson Companies Inc.,DPHIQ,Delphi Corp.
 10/4/2005,LEN,Lennar Corp.
-10/3/2005,,,G,Gillette Co.
+10/3/2005,,,1028411Q,Gillette Co.
 10/3/2005,AMP,Ameriprise Financial,PVN,Providian Financial
-8/30/2005,CVH,Coventry Health Care Inc.,MAY,May Dept. Stores
-8/19/2005,PSA,Public Storage,DAL,Delta Air Lines
+8/30/2005,CVH,Coventry Health Care Inc.,987200Q,May Dept. Stores
+8/19/2005,PSA,Public Storage,DALRQ,Delta Air Lines
 8/15/2005,MUR,Murphy Oil,NXTL,Nextel Communications
-8/12/2005,VNO,Vornado Realty Trust,SDS,SunGard Data Systems
+8/12/2005,VNO,Vornado Realty Trust,764144Q,SunGard Data Systems
 8/11/2005,TSN,Tyson Foods,UCL,Unocal Corp.
 7/22/2005,WFT,Weatherford Int'l,TOY,"Toys ""R"" US"
-7/5/2005,DHI,D.R. Horton,VRTS,Veritas Software
-7/5/2005,STZ,Constellation Brands,GLK,Great Lakes Chemical
-3/28/2005,SHLD,Sears Holdings Corp.,S,Sears
+7/5/2005,DHI,D.R. Horton,3153670Q,Veritas Software
+7/5/2005,STZ,Constellation Brands,723588Q,Great Lakes Chemical
+3/28/2005,SHLD,Sears Holdings Corp.,605555Q,Sears
 3/15/2005,NOV,National Oilwell Varco,PWER,Power-One
-12/29/2004,XTO,XTO Energy,PSFT,PeopleSoft Inc.
+12/29/2004,XTO,XTO Energy,3029830Q,PeopleSoft Inc.
 12/20/2004,ASN,Archstone-Smith Trust,CR,Crane Co.
-12/20/2004,CBSS,Compass Bancshares,WOR,Worthington Industries
+12/20/2004,8206908Q,Compass Bancshares,WOR,Worthington Industries
 12/20/2004,NWSA,News Corp.,DLX,Deluxe Corp.
-12/3/2004,FSL/B,Freescale Semiconductor I,WIN,Winn-Dixie
-12/1/2004,LLL,L-3 Communications Holdin,WLP,WellPoint Health Networks
-11/1/2004,LH,Laboratory Corp of Americ,SOTR,SouthTrust Corp.
-10/27/2004,CIT,CIT Group,AWE,AT&T Wireless
-9/1/2004,COH,Coach Inc.,CF,Charter One Financial
+12/3/2004,FSL/B,Freescale Semiconductor I,WNDXQ,Winn-Dixie
+12/1/2004,LLL,L-3 Communications Holdin,502466Q,WellPoint Health Networks
+11/1/2004,LH,Laboratory Corp of Americ,2676187Q,SouthTrust Corp.
+10/27/2004,CITGQ,CIT Group,AWE,AT&T Wireless
+9/1/2004,COH,Coach Inc.,605018Q,Charter One Financial
 8/3/2004,FSH,Fisher Scientific,TNB,Thomas & Betts
-7/1/2004,GILD,Gilead Sciences,ONE,Bank One Corp.
-7/1/2004,RF,Regions Financial Corp. (,RF,Regions Financial Corporation
+7/1/2004,GILD,Gilead Sciences,3621240Q,Bank One Corp.
+7/1/2004,RF,Regions Financial Corp. (,355488Q,Regions Financial Corporation
 7/1/2004,SOV,Sovereign Bancorp,UPC,Union Planters Corporation
 5/3/2004,HSP,Hospira Inc.,AM,American Greetings
 4/29/2004,VLO,Valero Energy,JHF,John Hancock Financial Servic
-4/23/2004,MYL,Mylan Laboratories,PCS,Sprint PCS
-4/2/2004,ACS,Affiliated Computer Servi,TAP/B,Travelers Property Casualty
+4/23/2004,MYL,Mylan Laboratories,945852Q,Sprint PCS
+4/2/2004,8277831Q,Affiliated Computer Servi,TAP/B,Travelers Property Casualty
 4/1/2004,ETFC,E*Trade Financial,FBF,FleetBoston Financial
 3/25/2004,CMX,Caremark Rx,TUP,Tupperware Corp.
-2/27/2004,MTB,M&T Bank Corp.,CE,Concord EFS Inc.
+2/27/2004,MTB,M&T Bank Corp.,519239Q,Concord EFS Inc.
 11/13/2003,BIIB,BIOGEN IDEC,BGEN,Biogen Inc.
 9/25/2003,ESRX,Express Scripts,QTRN,Quintiles Transnational
 8/20/2003,MHS,Medco Health Solutions In,MDR,McDermott International
-7/17/2003,8394653Q,ProLogis,MIR,Mirant Corp
+7/17/2003,8394653Q,ProLogis,MIRKQ,Mirant Corp
 4/16/2003,FII,Federated Investors Inc.,748957Q,Pharmacia Corp.
-3/31/2003,SYMC,Symantec Corp.,HI,Household International
-3/21/2003,MKC,McCormick & Company,HRC,HEALTHSOUTH Corp.
+3/31/2003,SYMC,Symantec Corp.,2727195Q,Household International
+3/21/2003,MKC,McCormick & Company,HLS,HEALTHSOUTH Corp.
 3/13/2003,AIV,Apartment Investment & Ma,AMR,AMR Corp.
-12/12/2002,DGX,Quest Diagnostics,TRW,TRW Inc.
+2/24/2003,AN,AutoNation Inc.,RATL,Rational Software
+12/12/2002,DGX,Quest Diagnostics,297504Q,TRW Inc.
 11/19/2002,CMCSA,Comcast Corp.,CMCSK,Comcast Class A Special
 9/4/2002,RAI,RJ Reynolds Tobacco
 9/2/2002,,,COC,Conoco Inc.
 8/21/2002,TAP/B,Travelers Property Casual,VTSS,Vitesse Semiconductor
 8/14/2002,MON,Monsanto Co,PALM,Palm Inc
-7/25/2002,WLP,Anthem Inc.,CNC,Conseco Inc.
-7/22/2002,EBAY,eBay Inc.,AL,Alcan Inc.
+7/25/2002,WLP,Anthem Inc.,CNCEQ,Conseco Inc.
+7/22/2002,EBAY,eBay Inc.,8214761Q,Alcan Inc.
 7/22/2002,ERTS,Electronic Arts,PDG,Placer Dome Inc.
 7/22/2002,GS,Goldman Sachs Group,UN,Unilevr N.V.
 7/22/2002,PFG,Principal Financial Group,ABX,Barrick Gold Corp.
-7/22/2002,PRU,Prudential Financial,NT,Nortel Networks
-7/22/2002,SDS,SunGard Data Systems,N,Inco
-7/22/2002,UPS,United Parcel Services,RD,Royal Dutch Petroleum
+7/22/2002,PRU,Prudential Financial,NRTLQ,Nortel Networks
+7/22/2002,764144Q,SunGard Data Systems,INCLF,Inco
+7/22/2002,UPS,United Parcel Services,RDPL,Royal Dutch Petroleum
 7/17/2002,NFB,North Fork Bancorporation,IMNX,Immunex Corp.
 6/26/2002,SPG,Simon Property Group,CNXT,Conexant Systems
-5/15/2002,APOL,Apollo Group,WCOM,WorldCom Inc.-WorldCom Group
-5/15/2002,BJS,BJ Services,U,USAirways Group Inc.
+5/15/2002,APOL,Apollo Group,WCOEQ,WorldCom Inc.-WorldCom Group
+5/15/2002,BJS,BJ Services,UAWGQ,USAirways Group Inc.
 5/13/2002,TT,American Standard,SAPE,Sapient Corp.
 5/6/2002,FHN,First Tennessee National,680884Q,COMPAQ Computer
-2/24/2002,AN,AutoNation Inc.,RATL,Rational Software
-2/11/2002,MI,Marshall & Ilsley Corp.,WLL,Willamette Industries
+2/11/2002,MI,Marshall & Ilsley Corp.,228510Q,Willamette Industries
 2/1/2002,RATL,Rational Software,NMK,Niagara Mohawk Hldgs. Inc.
-1/31/2002,MWV,MeadWestvaco Corp.,MEA,Mead Corp.
+1/31/2002,MWV,MeadWestvaco Corp.,544791Q,Mead Corp.
 1/30/2002,ACE,ACE Limited,W,Westvaco Corp.
-1/17/2002,PCL,Plum Creek Timber Co.,KM,K mart
+1/17/2002,PCL,Plum Creek Timber Co.,KMRTQ,K mart
 1/2/2002,WAT,Waters Corporation,FMC,FMC Corp.
 12/17/2001,JNY,Jones Apparel Group,HM,Homestake Mining
 12/14/2001,GENZ,Genzyme Corp.
 12/13/2001,,,RAL,Ralston Purina Co
-12/3/2001,EQR,Equity Res.Properties,NSI,National Service Ind.
-11/30/2001,NVDA,NVIDIA Corp.,ENE,Enron Corp.
+12/3/2001,EQR,Equity Res.Properties,329556Q,National Service Ind.
+11/30/2001,NVDA,NVIDIA Corp.,ENRNQ,Enron Corp.
 11/7/2001,HMA,Health Management Assoc.,GPUI,GPU Inc.
-10/10/2001,EOP,Equity Office Properties,TX,Texaco Inc.
+10/10/2001,EOP,Equity Office Properties,677004Q,Texaco Inc.
+10/10/2001,TE,TECO ENERGY INC,GBLXQ,GLOBAL CROSSING LTD
 9/21/2001,IMNX,Immunex Corp.
 9/18/2001,,,TOS,Tosco Corp.
 9/4/2001,IGT,Int'l. Game Technology,BVSN,BroadVision Inc.
 9/4/2001,XL,XL Capital,41018Q,Wachovia Corp.
 8/30/2001,ABC,AmerisourceBergen,OKE,ONEOK Inc.
-8/30/2001,CIEN,CIENA Corp.,AGC,American General
+8/30/2001,CIEN,CIENA Corp.,950967Q,American General
 8/7/2001,ZMH,Zimmer Holdings,TKR,Timken Co.
-8/6/2001,FDO,Family Dollar Stores,CS,Cabletron Systems
+8/6/2001,FDO,Family Dollar Stores,ETS,Cabletron Systems
 8/3/2001,PMCS,PMC-Sierra,OAT,Quaker Oats
 7/9/2001,AWE,AT&T Wireless Group,PCH,Potlatch Corp.
 7/2/2001,COL,Rockwell Collins,LDG,Longs Drug Stores
-6/28/2001,JHF,John Hancock Financial,H,Harcourt General Inc.
+6/28/2001,JHF,John Hancock Financial,739306Q,Harcourt General Inc.
 6/25/2001,,,AZA,ALZA Corp. Cl. A
 6/18/2001,ZION,Zions Bancorp
-6/4/2001,MWW,TMP Worldwide,CIT,CIT Group Inc.
-5/14/2001,PBG,Pepsi Bottling Group,ADPT,Adaptec
-4/3/2001,MIR,Mirant Corp.,BGG,Briggs & Stratton
-4/2/2001,CE,Concord EFS Inc.,CEN,Ceridian Corp (Old)
+6/4/2001,MWW,TMP Worldwide,89236Q,CIT Group Inc.
+5/14/2001,PBG,Pepsi Bottling Group,ADPTD,Adaptec
+4/3/2001,MIRKQ,Mirant Corp.,BGG,Briggs & Stratton
+4/2/2001,519239Q,Concord EFS Inc.,ARB,Ceridian Corp (Old)
 4/2/2001,FISV,FIserv Inc.,OK,Old Kent Financial
-3/1/2001,CTAS,Cintas Corporation,SUB,Summit Bancorp
+3/1/2001,CTAS,Cintas Corporation,3075558Q,Summit Bancorp
 2/27/2001,FTR,Citizens Communications,3346Q,U.S. Bancorp
 2/7/2001,UVN,Univision Communications,UK,Union Carbide
 1/30/2001,JBL,Jabil Circuit,CGP,Coastal Corp.
-1/16/2001,NE,Noble Drilling Corp.,TWX,Time Warner Inc.
+1/16/2001,NE,Noble Drilling Corp.,253647Q,Time Warner Inc.
 1/2/2001,AMCC,Applied Micro Circuits,3277Q,Morgan (J.P.) & Co.
 12/22/2000,FLR,Fluor Corp. (New),MEE,Massey Energy Company
-12/14/2000,AET,Aetna Inc. (New),AET,Aetna Inc. (Old)
-12/12/2000,KMI,Kinder Morgan,NGH,Nabisco Group Hldgs.
+12/14/2000,AET,Aetna Inc. (New),3253Q,Aetna Inc. (Old)
+12/12/2000,4621824Q,Kinder Morgan,NGH,Nabisco Group Hldgs.
 12/12/2000,QLGC,QLogic Corp.,PRDCQ,Polaroid Corp.
 12/12/2000,SYK,Stryker Corp.,RML,Russell Corp.
-12/12/2000,VTSS,Vitessa Semiconductor,SMI,Springs Industries
-12/11/2000,ABK,Ambac Financial Group,CCK,Crown Cork & Seal
-12/11/2000,AYE,Allegheny Energy Inc.,BS,Bethlehem Steel
+12/12/2000,VTSS,Vitessa Semiconductor,304024Q,Springs Industries
+12/11/2000,ABKFQ,Ambac Financial Group,CCK,Crown Cork & Seal
+12/11/2000,AYE,Allegheny Energy Inc.,BHMSQ,Bethlehem Steel
 12/11/2000,INTU,Intuit,OI,Owens-Illinois
-12/11/2000,MET,MetLife Inc.,VO,Seagram Co. Ltd.
+12/11/2000,MET,MetLife Inc.,291784Q,Seagram Co. Ltd.
 12/11/2000,2987536Q,Symbol Technologies,GRA,WR Grace & Co (New)
 12/5/2000,RHI,Robert Half International
 12/4/2000,,,AFS,Associates First Capital
-12/1/2000,CPN,Calpine Corp.,FPC,Florida Progress
+12/1/2000,CPNLQ,Calpine Corp.,FPC,Florida Progress
 11/24/2000,CHIR,Chiron Corp.,FJ,Fort James Corp.
 11/22/2000,FRX,Forest Laboratories,SEG,Seagate Technology
 11/17/2000,HOT,Starwood Hotels & Resorts,ACKH,Armstrong Holdings
-11/6/2000,BVSN,BroadVision Inc.,PWJ,PainWebber Group
-11/2/2000,EOG,EOG Resources,EFU,Eastern Enterprises
-11/2/2000,NI,NiSource Inc.,CG,Columbia Energy Group
+11/6/2000,BVSN,BroadVision Inc.,301710Q,PainWebber Group
+11/2/2000,EOG,EOG Resources,2269312Q,Eastern Enterprises
+11/2/2000,NI,NiSource Inc.,68335Q,Columbia Energy Group
 10/23/2000,PWER,Power-One Inc.,UCM,Unicom Corp.
 10/18/2000,NBR,Nabors Industries,MKG,Mallinckrodt Inc.
-10/3/2000,DYN,Dynegy Inc.,BFO,BestFoods
+10/3/2000,DYN,Dynegy Inc.,236555Q,BestFoods
 10/3/2000,KG,King Pharmaceuticals,YNR,Young & Rubicam
-10/2/2000,AV,Avaya Inc.,OWC,Owens Corning
-8/30/2000,DVN,Devon Energy Corp.,GAP,Great Atlantic & Pacific Tea
+10/2/2000,3414924Q,Avaya Inc.,OWENQ,Owens Corning
+8/30/2000,DVN,Devon Energy Corp.,GAPTQ,Great Atlantic & Pacific Tea
 8/21/2000,KSE,KeySpan Corp.,NCE,New Century Energies
 7/28/2000,PALM,Palm Inc.,COMS,3Com Corp.
 7/27/2000,JDSU,JDS Uniphase Corp.,RAD,Rite Aid Corp.
-7/17/2000,CIT,CIT Group Inc.,UPR,Union Pacific Resources
+7/17/2000,89236Q,CIT Group Inc.,2953349Q,Union Pacific Resources
 7/13/2000,JNS,Stilwell Financial,KSU,Kansas City Southern Industri
 7/6/2000,Q,Qwest Communication Int'l,USW,US West
-7/3/2000,BRCM,Broadcom Corporation,GTE,GTE Corp.
-6/29/2000,MERQ,Mercury Interactive,MZ,Milacron Inc.
-6/29/2000,VC,Visteon Corp.,IKN,IKON Office Solutions
+7/3/2000,BRCM,Broadcom Corporation,571300Q,GTE Corp.
+6/29/2000,MERQ,Mercury Interactive,MZIAQ,Milacron Inc.
+6/29/2000,VSTNQ,Visteon Corp.,IKN,IKON Office Solutions
 6/21/2000,SANM,Sanmina Corp.,WLA,Warner-Lambert
-6/21/2000,TIF,Tiffany & Co.,SGI,Silicon Graphics
-6/19/2000,CF,Charter One Financial
-6/19/2000,NVLS,Novellus Systems,CHA,Champion International
-6/16/2000,,,UMG,MediaOne Group Inc.
-6/16/2000,MEDI,MedImmune,CSR,Central & South West
+6/21/2000,TIF,Tiffany & Co.,SGID,Silicon Graphics
+6/19/2000,605018Q,Charter One Financial
+6/19/2000,NVLS,Novellus Systems,140402Q,Champion International
+6/16/2000,,,UMGI,MediaOne Group Inc.
+6/16/2000,MEDI,MedImmune,45200Q,Central & South West
 6/13/2000,CVG,Convergys Corp.,TMC,Times Mirror
-6/8/2000,SBUX,Starbucks Corp.,SMS,Shared Medical Systems
+6/8/2000,SBUX,Starbucks Corp.,2731112Q,Shared Medical Systems
 6/5/2000,A,Agilent Technologies,NC,NACCO Industries
-6/1/2000,APCC,American Power Conversion,MIR,Mirage Resorts
-5/10/2000,MXIM,Maxim Integrated Products,JOS,Jostens Inc.
+6/1/2000,2307532Q,American Power Conversion,3301Q,Mirage Resorts
+5/10/2000,MXIM,Maxim Integrated Products,JOSEA,Jostens Inc.
 5/5/2000,SAPE,Sapient Corp.,RLM,Reynolds Metals
-5/5/2000,SEBL,Siebel Systems,CBS,CBS Corp.
-4/18/2000,ALTR,Altera Corp.,ARC,Atlantic Richfield
+5/5/2000,SEBL,Siebel Systems,667517Q,CBS Corp.
+4/18/2000,ALTR,Altera Corp.,300583Q,Atlantic Richfield
 4/3/2000,LLTC,Linear Technology,PBY,Pep Boys
-4/3/2000,VRTS,Veritas Software,748957Q,Pharmacia & Upjohn
-3/16/2000,TSG,Sabre Holdings Corporatio,SRV,Service Corp. International
-1/31/2000,BGEN,Biogen,FWC,Foster Wheeler
-1/31/2000,CNXT,Conexant Systems,CNG,Consolidated Natural Gas
-1/31/2000,HOG,Harley-Davidson,FLE,Fleetwood Enterprises
+4/3/2000,3153670Q,Veritas Software,285939Q,Pharmacia & Upjohn
+3/16/2000,TSG,Sabre Holdings Corporatio,SCI,Service Corp. International
+1/31/2000,BGEN,Biogen,FWLT,Foster Wheeler
+1/31/2000,CNXT,Conexant Systems,CNRG,Consolidated Natural Gas
+1/31/2000,HOG,Harley-Davidson,FLTWQ,Fleetwood Enterprises
 1/6/2000,YNR,Young & Rubicam,827663Q,General Instrument
 1/4/2000,NCR,NCR Corp.
 10/15/1996,LGN,LOGICON INC,WHO,WATERHOUSE INVESTOR SERV
\ No newline at end of file
diff --git a/sp500 tickers 10-04.csv b/sp500 tickers 10-17.csv
index 1e2a0d9..7371056 100644
--- a/sp500 tickers 10-04.csv
+++ b/sp500 tickers 10-17.csv
@@ -94,8 +94,7 @@ Carnival Corp	CCL
 Caterpillar Inc	CAT 
 Celgene Corp	CELG 
 Centerpoint Energy Inc	CNP 
-CenturyLink Inc	CTL 
-Cephalon Inc	CEPH 
+CenturyLink Inc	CTL  
 Cerner Corp	CERN 
 Chesapeake Energy Corp	CHK 
 Chevron Corp	CVX 
@@ -432,6 +431,7 @@ Supervalu Inc	SVU
 Symantec Corp	SYMC 
 Sysco Corp	SYY 
 T Rowe Price Group Inc	TROW 
+TE Connectivity	   TEL
 TECO Energy Inc	TE 
 TJX Cos Inc	TJX 
 Target Corp	TGT 
diff --git a/getBloombergData.R b/utils.R
index 2421759..626324e 100644
--- a/getBloombergData.R
+++ b/utils.R
@@ -1,6 +1,11 @@
-getBloombergData <- function(conn,ticker,start.date){
-  require(TTR)
-  ohlc <- bdh(conn,paste(ticker,"Equity"),c("PX_OPEN","PX_HIGH","PX_LOW","PX_LAST","VOLUME"),start.date,dates.as.row.names=F)
+#fix bug in adjRatios
+source("patch-adjRatios.R")
+assignInNamespace("adjRatios",adjRatios,ns="TTR")
+library(TTR)
+rm(adjRatios)
+
+getBloombergData <- function(conn,ticker, start.date, end.date=Sys.Date()){
+  ohlc <- bdh(conn,paste(ticker,"Equity"),c("PX_OPEN","PX_HIGH","PX_LOW","PX_LAST","VOLUME"),start.date,end.date,dates.as.row.names=F)
   colnames(ohlc) <- c("Date","Open","High","Low","Close","Volume")
   ohlc <- xts(ohlc[,-1],as.Date(ohlc$Date))
   #split information
@@ -30,8 +35,8 @@ getBloombergData <- function(conn,ticker,start.date){
                order.by = as.Date(spl$"Adjustment Date"))
     #aggregate non-unique dates
     spl <- as.xts(aggregate(spl, identity, prod, na.rm=T))
-    if(any(time(spl)>= start.date & time(spl)<=Sys.Date())){
-      spl <- window(spl, start=start.date, end = Sys.Date())
+    if(any( (time(spl) >= start.date) && (time(spl) <= end.date))){
+      spl <- window(spl, start=start.date, end = end.date)
     }else{
       spl <- NULL
     }
@@ -40,12 +45,15 @@ getBloombergData <- function(conn,ticker,start.date){
   #we need to override the end date as well cause the Ex-Date might be in the
   #future
   override_fields <- c("DVD_START_DT", "DVD_END_DT")
-  override_values <- c(format(start.date,"%Y%m%d"),format(Sys.Date(),"%Y%m%d"))
+  override_values <- c(format(start.date, "%Y%m%d"),format(end.date, "%Y%m%d"))
   div <- bds(conn,paste(ticker,"Equity"), c("DVD_HIST"), override_fields,
              override_values)
   if(NROW(div)!=0){
     div <- xts(div$"Dividend Amount", as.Date(div$"Ex-Date"))
+    #aggregate non-unique dates
+    div <- as.xts(aggregate(div, identity, sum))
   }
+
   if(is.null(div) && is.null(spl)){
     divspl <- NULL
   }else if(is.null(div)){
@@ -88,3 +96,28 @@ getBloombergData <- function(conn,ticker,start.date){
   }
   return( ohlc )
 }
+
+memb <- function(index,add,date){
+  #return the spx tickers at a given date using the history of changes
+  #index: sp500 tickers as of today
+  #add: tabele with list of changes
+  #date: date at which we make the query
+  toreverse <- add[add$date>=date,]
+  current.index <- index
+  for(i in 1:nrow(toreverse)){
+    if(toreverse$ticker.add[i]!=""){
+      current.index <- current.index[-match(toreverse$ticker.add[i],current.index)]
+    }
+    if(toreverse$ticker.del[i]!=""){
+      current.index <- sort(c(current.index,toreverse$ticker.del[i]))
+    }
+  }
+  current.index
+}
+
+memb.bb <- function(date){
+  #return the spx tickers at a given date using bloomberg api
+  sp500.tickers <- bds(conn, "SPX Index", "INDX_MWEIGHT_HIST",
+      "END_DATE_OVERRIDE", format(date, "%Y%m%d"))
+  return( gsub(" [A-Z]*","",sp500.tickers[,1]) )
+}