Adding to my comment above, below is Scala code to parse the entire bitcoind blockchain and extract raw blocks. It uses Bitcoinj.jar library to further parse the raw block.
The blocks are stored in files blkxxxxx.dat. The structure of the file is as follows:
4 | 4 | 80 | TxData | 4 | 4 | 80 | TxData | 4 | 4 | 80 | TxData | ...
First 4 bytes: magic bytes (identifying which network you are on)
Second 4 bytes: the number of bytes of the remaining block
Next 80 bytes: block header itself
Next NumBlockBytes - 80 bytes: Transaction data in this block [ numTx | Tx1 | Tx2 | Tx3 | ... ]
In my system, I was able to iterate through all the files (1000+) within 4 hours (no verification or processing of block bytes, just the dummy code below). There was around 140 GB data on the blockchain at that time. Perhaps some Scala gurus can make it faster.
Interestingly, when I was syncing bitcoind for the first time, it finished within 6 hours which includes downloading and verifying the blocks. So this will be faster in C++.
Also you'll have to deal with orphans.
import java.io._
import java.nio._
import scala.collection.mutable.ArrayBuffer
import org.apache.commons.io.FileUtils
import org.bitcoinj.core._
import org.bitcoinj.params._
import scala.collection.JavaConversions._
object Utils {
// Used for closing files implicitly
def using[A <: {def close(): Unit}, B](param: A)(f: A => B): B = try { f(param) } finally { param.close() }
// this is the method that actually parses the file
def parseFile(name:String) = {
System.gc // large files (around 140 MB each, need to clear memory)
using(new FileInputStream(name)) {fis =>
using(new BufferedInputStream(fis)) {bis =>
var currBlkByte = -1 // which byte of raw block are we reading?
var currBlk = 0 // which block is currently being read?
var currBlkSize = -1L // what is the size of block (in bytes)
var endBlkByte = -1 // which is the ending byte of current block?
val blkSizeBytes = new ArrayBuffer[Byte] // stores bytes containing data about block size
val blkBytes = new ArrayBuffer[Byte] // stores bytes of block
Stream.continually(bis.read).takeWhile(-1 !=).foreach{int =>
currBlkByte += 1
val byte = int.toByte
// ignore first 4 bytes (magic bytes), next 4 bytes stores upcoming block's size in little endian
if (currBlkByte >= 4 && currBlkByte < 8) blkSizeBytes += byte
if (currBlkByte == 7) { // this byte is the last one encoding block's size
currBlkSize = ByteBuffer.wrap(blkSizeBytes.toArray).order(ByteOrder.LITTLE_ENDIAN).getInt & 0xFFFFFFFFL;
endBlkByte = currBlkSize.toInt + 7 // first 8 bytes for info, remaining encoding block
blkSizeBytes.clear // clear for next block
}
if (currBlkByte > 7) blkBytes += byte // block data
if (currBlkByte == endBlkByte) { // we have reached end of block
// last block byte
currBlk += 1 // increment block count
currBlkByte = -1 // reset
endBlkByte = -1 // reset
parseBlk(blkBytes.toArray) // we have block in bytes, lets parse it
blkBytes.clear // reset
}
}
}
}
}
val context = new Context(MainNetParams.get) // needed for Bitcoinj v 0.13 and above
def parseBlk(bytes:Array[Byte]) = { // uses Bitcoinj
new Block(MainNetParams.get, bytes).getTransactions.foreach {tx =>
val hash = tx.getHashAsString
val inputs = tx.getInputs
val outputs = tx.getOutputs
// do something with above
}
}
def getAllFiles(dir:String, extensions:Array[String], recursive:Boolean) =
FileUtils.listFiles(new File(dir), extensions, recursive).toArray.map(_.toString)
}
import Utils._
object BlockParser {
val dir = "/home/user/.bitcoin/blocks"
//files have names like blk00000.dat, ..., blk01096.dat (last one at time of writing)
val files = getAllFiles(dir, Array("dat"), false).collect {
case name if name.contains("blk") => // collect only those file with names like "blkxxxxx.dat"
val num = name.drop(s"$dir/blk".size).take(5).toInt // (take 5 is based on actual file names)
(name, num)
}.sortBy(_._2).unzip._1 // sort by file number
files.foreach(parseFile)
}
