What exactly is hashed and signed in a segwit transaction

Question

I have an unsigned raw segwit transaction on my regtest network:

02000000011333183ddf384da83ed49296136c70d206ad2b19331bf25d390e69b222165e370000000000ffffffff0200e1f5050000000017a914a860f76561c85551594c18eecceffaee8c4822d787f0c1a4350000000017a914d8b6fcc85a383261df05423ddf068a8987bf02878700000000

Pretty format:

Version: 02000000
Flag:
Input Count: 01
Input 1 Previous Output Hash: 1333183ddf384da83ed49296136c70d206ad2b19331bf25d390e69b222165e37
Input 1 Previous Output Index: 00000000
Input 1 script length: 00 (empty for now)
Input 1 scriptSig: 
Input 1 sequence: feffffff
Output Count: 02
Output 1 Value: 00e1f50500000000 (100M satoshis / 1 BTC)
Output 1 public key script length: 0x17 (23 bytes)
Output 1 public key script: a914a860f76561c85551594c18eecceffaee8c4822d787
Output 2 Value: F0C1A43500000000 (899990000 sats / 8.9999 BTC)
Output 2 public key script length: 0x17 (23 bytes)
Output 2 public key script: a914d8b6fcc85a383261df05423ddf068a8987bf028787
Witness Count:
Witness 1 length:
Witness 1: 
Witness 2 length:
Witness 2: 
Locktime: 8c000000 (block 140)

I can sign it using the above data (concatenated from top to bottom):

$ bitcoin-cli -regtest signrawtransaction 02000000011333183ddf384da83ed49296136c70d206ad2b19331bf25d390e69b222165e370000000000feffffff0200e1f5050000000017a914a860f76561c85551594c18eecceffaee8c4822d787F0C1A4350000000017a914d8b6fcc85a383261df05423ddf068a8987bf0287878c000000

{
  "hex": "020000000001011333183ddf384da83ed49296136c70d206ad2b19331bf25d390e69b222165e370000000017160014b93f973eb2bf0b614bddc0f47286788c98c535b4feffffff0200e1f5050000000017a914a860f76561c85551594c18eecceffaee8c4822d787f0c1a4350000000017a914d8b6fcc85a383261df05423ddf068a8987bf028787024730440220434caf5bb442cb6a251e8bce0ec493f9a1a9c4423bcfc029e542b0e8a89d1b3f022011090d4e98f79c62b188245a4aa4eb77e912bfd57e0a9b9a1c5e65f2b39f3ab401210223bec70d670d29a30d9bcee197910e37cf2a10f0dc3c5ac44d865aec0d7052fb8c000000",
  "complete": true
}

But, when I try to do it manually, I get a different result for the signature in witness. I think I may be signing the wrong data, so what would the serialized hex string look like that gets hashed (HASH256) and then signed to produce the signature in the Witness data (Witness 1 below)?

Final Tx (signed using signrawtransaction)

Version: 02000000
Flag: 0001 (indicates segwit)
Input Count: 01
Input 1 Previous Output Hash: 1333183ddf384da83ed49296136c70d206ad2b19331bf25d390e69b222165e37
Input 1 Previous Output Index: 00000000
Input 1 script length: 0x17 (23 bytes)
Input 1 scriptSig: 160014b93f973eb2bf0b614bddc0f47286788c98c535b4
Input 1 sequence: feffffff
Output Count: 02
Output 1 Value: 00e1f50500000000 (100M satoshis / 1 BTC)
Output 1 public key script length: 0x17 (23 bytes)
Output 1 public key script: a914a860f76561c85551594c18eecceffaee8c4822d787
Output 2 Value: F0C1A43500000000 (899990000 sats / 8.9999 BTC)
Output 2 public key script length: 0x17 (23 bytes)
Output 2 public key script: a914d8b6fcc85a383261df05423ddf068a8987bf028787
Witness Count: 02
Witness 1 length: 0x47 (71 bytes)
Witness 1: 30440220434caf5bb442cb6a251e8bce0ec493f9a1a9c4423bcfc029e542b0e8a89d1b3f022011090d4e98f79c62b188245a4aa4eb77e912bfd57e0a9b9a1c5e65f2b39f3ab401
Witness 2 length: 0x21 (33 bytes)
Witness 2: 0223bec70d670d29a30d9bcee197910e37cf2a10f0dc3c5ac44d865aec0d7052fb
Locktime: 8c000000 (block 140)

Edit: Code for signing: GIST - ec_sign_hex.sh. This takes the final hashed hex string and a 32-byte hex private key as parameters. The private key I'm using is 72b90220501a0c27a499535c010de9e2fa190c0d287a940a3886bcb2cbcccdb8, and it's executed as:

$ ec_sign_hex 02000000011333183ddf384da83ed49296136c70d206ad2b19331bf25d390e69b222165e370000000000feffffff0200e1f5050000000017a914a860f76561c85551594c18eecceffaee8c4822d787F0C1A4350000000017a914d8b6fcc85a383261df05423ddf068a8987bf0287878c000000 72b90220501a0c27a499535c010de9e2fa190c0d287a940a3886bcb2cbcccdb8

#!/bin/bash
## Command Line parsing
#######################

if [[ $# -lt 2 ]]; then
    echo "Usage: $ ec_sign_hex <input-hex> <priv-key-hex>"
    exit 1
fi

inputHex=$1
privKeyHex=$2

## Create .pem and .pub files
#############################
pubKeyHex="$(openssl ec -inform DER -text -noout -in <(cat <(echo -n "302e0201010420") <(echo -n "${privKeyHex}") <(echo -n "a00706052b8104000a") | xxd -r -p) 2>/dev/null | tail -6 | head -5 | sed 's/[ :]//g' | tr -d '\n')"
asnFormatKey="30740201010420${privKeyHex}a00706052b8104000aa144034200${pubKeyHex}"
echo "-----BEGIN EC PRIVATE KEY-----" > tmp.pem
echo $asnFormatKey | xxd -r -p | base64 | fold -w 64 >> tmp.pem
echo "-----END EC PRIVATE KEY-----" >> tmp.pem

openssl ec -in tmp.pem -pubout -out tmpPub.pem &>/dev/null

## Sign message
#  sign:
    openssl pkeyutl -sign -inkey tmp.pem -in <(printf $inputHex | xxd -r -p) -out tmp.sig
    echo "Signature"
    echo "####################"
    echo ""
    openssl pkeyutl -sign -inkey tmp.pem -in <(printf $inputHex | xxd -r -p) | xxd -p #-hexdump #| xxd -p
    echo ""
    echo "####################"
#  verify:
    openssl pkeyutl -verify -pubin -inkey tmpPub.pem -sigfile tmp.sig -in <(printf $inputHex | xxd -r -p)

rm tmp.pem tmpPub.pem tmp.sig

Update: I believe the above script isn't working because pkeyutl defaults to a sha1 fingerprint before signing.

Answer 1

Figured out a few things:

Segwit Signatures

For segwit transactions one must sign the following (see BIP 143):

Double SHA256 of the serialization of:
     1. nVersion of the transaction (4-byte little endian)
     2. hashPrevouts (32-byte hash)
     3. hashSequence (32-byte hash)
     4. outpoint (32-byte hash + 4-byte little endian) 
     5. scriptCode of the input (serialized as scripts inside CTxOuts)
     6. value of the output spent by this input (8-byte little endian)
     7. nSequence of the input (4-byte little endian)
     8. hashOutputs (32-byte hash)
     9. nLocktime of the transaction (4-byte little endian)
    10. sighash type of the signature (4-byte little endian)

For the example above, a P2SH-P2WPKH, that would be:

hash prevouts - hash256(1333183ddf384da83ed49296136c70d206ad2b19331bf25d390e69b222165e3700000000)
6623eab09650c6a6a98617d581c5d1fc26c6f5f158820e68ee636be93b433cee

hashSequence - hash256(feffffff)
18606b350cd8bf565266bc352f0caddcf01e8fa789dd8a15386327cf8cabe198

hashOutputs - hash256(00e1f5050000000017a914a860f76561c85551594c18eecceffaee8c4822d787F0C1A4350000000017a914d8b6fcc85a383261df05423ddf068a8987bf028787)
6a7522acd2fc865421d71893d71027bc3ebe0839d25dabc6277aef64b24b2370

hashPreimage
020000006623eab09650c6a6a98617d581c5d1fc26c6f5f158820e68ee636be93b433cee18606b350cd8bf565266bc352f0caddcf01e8fa789dd8a15386327cf8cabe1981333183ddf384da83ed49296136c70d206ad2b19331bf25d390e69b222165e37000000001976a914b93f973eb2bf0b614bddc0f47286788c98c535b488ac00ca9a3b00000000feffffff6a7522acd2fc865421d71893d71027bc3ebe0839d25dabc6277aef64b24b23708c00000001000000

nVersion:     02000000
hashPrevouts: 6623eab09650c6a6a98617d581c5d1fc26c6f5f158820e68ee636be93b433cee
hashSequence: 18606b350cd8bf565266bc352f0caddcf01e8fa789dd8a15386327cf8cabe198
outpoint:     1333183ddf384da83ed49296136c70d206ad2b19331bf25d390e69b222165e3700000000
scriptCode:   (OP_DUP OP_HASH160 <pubkey hash> OP_EQUALVERIFY OP_CHECKSIG) where <pubkey hash> is from redeemscript from previous output
              1976a914b93f973eb2bf0b614bddc0f47286788c98c535b488ac
amount:       00ca9a3b00000000 (where amount is the total utxo amount)
nSequence:    feffffff
hashOutputs:  6a7522acd2fc865421d71893d71027bc3ebe0839d25dabc6277aef64b24b2370
nLockTime:    8c000000
nHashType:    01000000 (SIGHASH_ALL)

The hashPreimage is double sha256 hashed to produce:

ef7d163bfe970439476d15537e1373bc23bb6282b9e145115331975a1a673788, which is the data to be signed. This results in a signature: 30440220434caf5bb442cb6a251e8bce0ec493f9a1a9c4423bcfc029e542b0e8a89d1b3f022011090d4e98f79c62b188245a4aa4eb77e912bfd57e0a9b9a1c5e65f2b39f3ab401.

Signature Script

However, the signature script above would never work didn't match because openssl cli uses a random nonce each time you sign, see https://bitcoin.stackexchange.com/a/32962/60443. So I wrote a small cli tool using libsecp256k1 utilized in bitcoin. Compiles using:

g++ -std=c++11 -o ec_sign ec_sign.cpp $(pkg-config --cflags libsecp256k1 --libs libsecp256k1)

Usage: ec_sign <hash-to-sign> <priv-key> <hash-type>(default 1: SIGHASH_ALL)

#include <secp256k1.h>
#include <iostream>
#include <iomanip>

int char2int(char input)
{
  if(input >= '0' && input <= '9')
    return input - '0';
  if(input >= 'A' && input <= 'F')
    return input - 'A' + 10;
  if(input >= 'a' && input <= 'f')
    return input - 'a' + 10;
  throw std::invalid_argument("Invalid input string");
}

void hex2bin(const char* src, char* target)
{
  while(*src && src[1])
  {
    *(target++) = char2int(*src)*16 + char2int(src[1]);
    src += 2;
  }
}

// EC sign/verify
// ----------------------------------------------------------------------------

int main(int argc, char *argv[])
{
    uint8_t hashType = 1;
    if(argc < 3)
    {
        std::cerr << "Usage: ec_sign <hash-to-sign> <priv-key> <hash-type>(default 1: SIGHASH_ALL)" << std::endl;
        return 1;
    }
    if(argc == 4)
    {
        hashType = argv[3][0] - '0';
    }
    char hash[32];
    hex2bin(argv[1], hash);
    char secret[32];
    hex2bin(argv[2], secret);

    size_t siglen = 71;
    secp256k1_ecdsa_signature signature;
    static secp256k1_context *ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY);

    unsigned char derSig[71];

    if (secp256k1_ecdsa_sign(ctx, &signature, (const unsigned char*)hash, (const unsigned char*)secret, secp256k1_nonce_function_rfc6979, NULL) != 1)
    {
        std::cerr << "Signature failed" << std::endl;
        return 1;
    }

    if (secp256k1_ecdsa_signature_serialize_der(ctx, derSig, &siglen, &signature) != 1)
    {
        std::cerr << "Serialization failed" << std::endl;
        return 1;
    }

    // Apply Hash type
    derSig[70] = hashType;

    for(int i=0; i<71; i++)
    {
        std::cout << std::hex << std::setfill('0') << std::setw(2) << (int)derSig[i];
    }
    std::cout << std::endl;

    return 0;
}