Fixed a bug in reading varbinary max fields (#1209)

This commit is contained in:
Jenny Tam 2020-10-23 14:21:14 -07:00 committed by GitHub
commit a161c33989
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 500 additions and 68 deletions

View file

@ -882,7 +882,7 @@ SQLRETURN binary_to_string( _Inout_ SQLCHAR* field_data, _Inout_ SQLLEN& read_so
_In_ SQLLEN buffer_length, _Inout_ SQLLEN* out_buffer_length,
_Inout_ sqlsrv_error_auto_ptr& out_error )
{
// hex characters for the conversion loop below
// The hex characters for the conversion loop below
static char hex_chars[] = "0123456789ABCDEF";
SQLSRV_ASSERT( out_error == 0, "Pending error for sqlsrv_buffered_results_set::binary_to_string" );
@ -892,17 +892,19 @@ SQLRETURN binary_to_string( _Inout_ SQLCHAR* field_data, _Inout_ SQLLEN& read_so
// Set the amount of space necessary for null characters at the end of the data.
SQLSMALLINT extra = sizeof(Char);
SQLSRV_ASSERT( ((buffer_length - extra) % (extra * 2)) == 0, "Must be multiple of 2 for binary to system string or "
"multiple of 4 for binary to wide string" );
// TO convert a binary to a system string or a binary to a wide string, the buffer size minus
// 'extra' is ideally multiples of 2 or 4 (depending on Char), but calculating to_copy_hex below
// takes care of this.
// all fields will be treated as ODBC returns varchar(max) fields:
// All fields will be treated as ODBC returns varchar(max) fields:
// the entire length of the string is returned the first
// call in out_buffer_len. Successive calls return how much is
// left minus how much has already been read by previous reads
// *2 is for each byte to hex conversion and * extra is for either system or wide string allocation
// *2 is for each byte to hex conversion and * extra is for either system
// or wide string allocation
*out_buffer_length = (*reinterpret_cast<SQLLEN*>( field_data - sizeof( SQLULEN )) - read_so_far) * 2 * extra;
// copy as much as we can into the buffer
// Will copy as much as we can into the buffer
SQLLEN to_copy;
if( buffer_length < *out_buffer_length + extra ) {
to_copy = (buffer_length - extra);
@ -915,14 +917,14 @@ SQLRETURN binary_to_string( _Inout_ SQLCHAR* field_data, _Inout_ SQLLEN& read_so
to_copy = *out_buffer_length;
}
// if there are bytes to copy as hex
// If there are bytes to copy as hex
if( to_copy > 0 ) {
// quick hex conversion routine
Char* h = reinterpret_cast<Char*>( buffer );
BYTE* b = reinterpret_cast<BYTE*>( field_data );
Char* h = reinterpret_cast<Char*>(buffer);
BYTE* b = reinterpret_cast<BYTE*>(field_data + read_so_far);
// to_copy contains the number of bytes to copy, so we divide the number in half (or quarter)
// to get the number of hex digits we can copy
SQLLEN to_copy_hex = to_copy / (2 * extra);
// to get the maximum number of hex digits to copy
SQLLEN to_copy_hex = static_cast<SQLLEN>(floor(to_copy / (2 * extra)));
for( SQLLEN i = 0; i < to_copy_hex; ++i ) {
*h = hex_chars[(*b & 0xf0) >> 4];
h++;
@ -930,7 +932,7 @@ SQLRETURN binary_to_string( _Inout_ SQLCHAR* field_data, _Inout_ SQLLEN& read_so
h++;
}
read_so_far += to_copy_hex;
*h = static_cast<Char>( 0 );
*h = static_cast<Char>(0);
}
else {
reinterpret_cast<char*>( buffer )[0] = '\0';

View file

@ -101,13 +101,18 @@ size_t sqlsrv_stream_read(_Inout_ php_stream* stream, _Out_writes_bytes_(count)
throw core::CoreException();
}
// if the stream returns either no data, NULL data, or returns data < than the count requested then
// we are at the "end of the stream" so we mark it
if( r == SQL_NO_DATA || read == SQL_NULL_DATA || ( static_cast<size_t>( read ) <= count && read != SQL_NO_TOTAL )) {
// If the stream returns no data or NULL data, mark the "end of the stream" and return
if( r == SQL_NO_DATA || read == SQL_NULL_DATA) {
stream->eof = 1;
return 0;
}
// If the stream returns data less than the count requested then we are at the "end of the stream" but continue processing
if (static_cast<size_t>(read) <= count && read != SQL_NO_TOTAL) {
stream->eof = 1;
}
// if ODBC returns the 01004 (truncated string) warning, then we return the count minus the null terminator
// If ODBC returns the 01004 (truncated string) warning, then we return the count minus the null terminator
// if it's not a binary encoded field
if( r == SQL_SUCCESS_WITH_INFO ) {
@ -120,26 +125,42 @@ size_t sqlsrv_stream_read(_Inout_ php_stream* stream, _Out_writes_bytes_(count)
SQLSRV_ASSERT( is_truncated_warning( state ), "sqlsrv_stream_read: truncation warning was expected but it "
"did not occur." );
}
// with unixODBC connection pooling enabled the truncated state may not be returned so check the actual length read
// with buffer length.
// As per SQLGetData documentation, if the length of character data exceeds the BufferLength,
// SQLGetData truncates the data to BufferLength less the length of null-termination character.
// But when fetching binary fields as chars (wide chars), each byte is represented as 2 hex characters,
// each takes the size of a char (wide char). Note that BufferLength may not be multiples of 2 or 4.
bool is_binary = (ss->sql_type == SQL_BINARY || ss->sql_type == SQL_VARBINARY || ss->sql_type == SQL_LONGVARBINARY);
// With unixODBC connection pooling enabled the truncated state may not be returned so check the actual length read
// with buffer length.
#ifndef _WIN32
if( is_truncated_warning( state ) || count < read) {
#else
if( is_truncated_warning( state ) ) {
#endif // !_WIN32
size_t char_size = sizeof(SQLCHAR);
switch( c_type ) {
// As per SQLGetData documentation, if the length of character data exceeds the BufferLength,
// SQLGetData truncates the data to BufferLength less the length of null-termination character.
case SQL_C_BINARY:
read = count;
break;
case SQL_C_WCHAR:
read = ( count % 2 == 0 ? count - 2 : count - 3 );
char_size = sizeof(SQLWCHAR);
if (is_binary) {
// Each binary byte read will be 2 hex wide chars in the buffer
SQLLEN num_bytes_read = static_cast<SQLLEN>(floor((count - char_size) / (2 * char_size)));
read = num_bytes_read * char_size * 2 ;
} else {
read = (count % 2 == 0 ? count - 2 : count - 3);
}
break;
case SQL_C_CHAR:
read = count - 1;
if (is_binary) {
read = ((count - char_size) % 2 == 0 ? count - char_size : count - char_size - 1);
} else {
read = count - 1;
}
break;
default:
DIE( "sqlsrv_stream_read: should have never reached in this switch case.");
@ -151,10 +172,10 @@ size_t sqlsrv_stream_read(_Inout_ php_stream* stream, _Out_writes_bytes_(count)
}
}
// if the encoding is UTF-8
// If the encoding is UTF-8
if( c_type == SQL_C_WCHAR ) {
count *= 2;
// undo the shift to use the full buffer
// Undo the shift to use the full buffer
// flags set to 0 by default, which means that any invalid characters are dropped rather than causing
// an error. This happens only on XP.
// convert to UTF-8

View file

@ -0,0 +1,153 @@
--TEST--
Test fetching varbinary, varchar, nvarchar max fields with client buffer
--DESCRIPTION--
Similar to sqlsrv_fetch_large_stream test but fetching varbinary, varchar, nvarchar max fields as strings with or without client buffer
--SKIPIF--
<?php require_once('skipif_mid-refactor.inc'); ?>
--ENV--
PHPT_EXEC=true
--FILE--
<?php
require_once("MsCommon_mid-refactor.inc");
$tableName = 'pdoFetchLobTest';
$binaryColumn = 'varbinary_max';
$strColumn = 'varchar_max';
$nstrColumn = 'nvarchar_max';
$bin = 'abcdefghijklmnopqrstuvwxyz';
$binaryValue = str_repeat($bin, 100);
$hexValue = str_repeat(strtoupper(bin2hex($bin)), 100);
$strValue = str_repeat("stuvwxyz", 400);
$nstrValue = str_repeat("ÃÜðßZZýA©", 200);
function checkData($actual, $expected)
{
trace("Actual:\n$actual\n");
$success = true;
$pos = strpos($actual, $expected);
if (($pos === false) || ($pos > 1)) {
$success = false;
}
return ($success);
}
function fetchBinary($conn, $buffered)
{
global $tableName, $binaryColumn, $binaryValue, $hexValue;
try {
$query = "SELECT $binaryColumn FROM $tableName";
if ($buffered) {
$stmt = $conn->prepare($query, array(PDO::ATTR_CURSOR=>PDO::CURSOR_SCROLL, PDO::SQLSRV_ATTR_CURSOR_SCROLL_TYPE=>PDO::SQLSRV_CURSOR_BUFFERED));
} else {
$stmt = $conn->prepare($query);
}
$stmt->bindColumn($binaryColumn, $value, PDO::PARAM_LOB, 0, PDO::SQLSRV_ENCODING_BINARY);
$stmt->execute();
$row = $stmt->fetch(PDO::FETCH_BOUND);
if (!checkData($value, $binaryValue)) {
echo "Fetched binary value unexpected ($buffered): $value\n";
}
$stmt->bindColumn($binaryColumn, $value, PDO::PARAM_LOB, 0, PDO::SQLSRV_ENCODING_SYSTEM);
$stmt->execute();
$row = $stmt->fetch(PDO::FETCH_BOUND);
if (!checkData($value, $hexValue)) {
echo "Fetched binary value a char string ($buffered): $value\n";
}
$stmt->bindColumn($binaryColumn, $value, PDO::PARAM_LOB, 0, PDO::SQLSRV_ENCODING_UTF8);
$stmt->execute();
$row = $stmt->fetch(PDO::FETCH_BOUND);
if (!checkData($value, $hexValue)) {
echo "Fetched binary value as UTF-8 string ($buffered): $value\n";
}
} catch (PdoException $e) {
echo "Caught exception in fetchBinary ($buffered):\n";
echo $e->getMessage() . PHP_EOL;
}
}
function fetchAsString($conn, $buffered)
{
global $tableName, $strColumn, $strValue;
global $nstrColumn, $nstrValue;
try {
$query = "SELECT $strColumn, $nstrColumn FROM $tableName";
if ($buffered) {
$stmt = $conn->prepare($query, array(PDO::ATTR_CURSOR=>PDO::CURSOR_SCROLL, PDO::SQLSRV_ATTR_CURSOR_SCROLL_TYPE=>PDO::SQLSRV_CURSOR_BUFFERED));
} else {
$stmt = $conn->prepare($query);
}
$stmt->execute();
$stmt->bindColumn($strColumn, $value1, PDO::PARAM_STR);
$stmt->bindColumn($nstrColumn, $value2, PDO::PARAM_STR);
$row = $stmt->fetch(PDO::FETCH_BOUND);
if (!checkData($value1, $strValue)) {
echo "Fetched string value ($buffered): $value1\n";
}
if (!checkData($value2, $nstrValue)) {
echo "Fetched string value ($buffered): $value2\n";
}
$stmt->execute();
$stmt->bindColumn($strColumn, $value, PDO::PARAM_STR, 0, PDO::SQLSRV_ENCODING_SYSTEM);
$row = $stmt->fetch(PDO::FETCH_BOUND);
if (!checkData($value, $strValue)) {
echo "Fetched string value: $value\n";
}
} catch (PdoException $e) {
echo "Caught exception in fetchBinary ($buffered):\n";
echo $e->getMessage() . PHP_EOL;
}
}
try {
$conn = connect();
$conn->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
// Create table of one max column
$colMeta = array(new ColumnMeta('varbinary(max)', $binaryColumn),
new ColumnMeta('varchar(max)', $strColumn),
new ColumnMeta('nvarchar(max)', $nstrColumn));
createTable($conn, $tableName, $colMeta);
// Insert one row
$query = "INSERT INTO $tableName ($binaryColumn, $strColumn, $nstrColumn) VALUES (?, ?, ?)";
$stmt = $conn->prepare($query);
$stmt->bindParam(1, $binaryValue, PDO::PARAM_LOB, 0, PDO::SQLSRV_ENCODING_BINARY);
$stmt->bindParam(2, $strValue, PDO::PARAM_STR, 0, PDO::SQLSRV_ENCODING_SYSTEM);
$stmt->bindParam(3, $nstrValue, PDO::PARAM_STR);
$stmt->execute();
unset($stmt);
// Starting fetching with or without client buffer
fetchBinary($conn, false);
fetchBinary($conn, true);
fetchAsString($conn, false);
fetchAsString($conn, true);
dropTable($conn, $tableName);
echo "Done\n";
unset($conn);
} catch (PdoException $e) {
echo $e->getMessage() . PHP_EOL;
}
?>
--EXPECT--
Done

View file

@ -1,55 +1,113 @@
--TEST--
Streaming Field Test
Test fetching varchar and nvarchar max fields
--DESCRIPTION--
Verifies the streaming behavior and proper error handling with Always Encrypted
Test fetching varchar and nvarchar max fields as streams or strings with or without client buffer
--SKIPIF--
<?php require('skipif_versions_old.inc'); ?>
--ENV--
PHPT_EXEC=true
--FILE--
<?php
require_once('MsCommon.inc');
$conn = AE\connect();
$tableName = "test_max_fields";
AE\createTable($conn, $tableName, array(new AE\ColumnMeta("varchar(max)", "varchar_max_col")));
$tableName = "char_max_fields";
$columns = array(new AE\ColumnMeta("varchar(max)", "varchar_max_col"),
new AE\ColumnMeta("nvarchar(max)", "nvarchar_max_col"));
$inValue = str_repeat("ÃÜðßZZýA©", 600);
$insertSql = "INSERT INTO $tableName (varchar_max_col) VALUES (?)";
$params = array($inValue);
AE\createTable($conn, $tableName, $columns);
$strValue = str_repeat("SimpleTest", 450);
$nstrValue = str_repeat("ÃÜðßZZýA©", 600);
$insertSql = "INSERT INTO $tableName (varchar_max_col, nvarchar_max_col) VALUES (?, ?)";
$params = array(array($strValue, null, null, SQLSRV_SQLTYPE_VARCHAR('max')),
array($nstrValue, null, SQLSRV_PHPTYPE_STRING('UTF-8'), SQLSRV_SQLTYPE_NVARCHAR('max')));
$stmt = sqlsrv_prepare($conn, $insertSql, $params);
if ($stmt) {
sqlsrv_execute($stmt);
}
$query = "SELECT * FROM $tableName";
$stmt = sqlsrv_prepare($conn, $query);
if ($stmt) {
sqlsrv_execute($stmt);
}
if (!sqlsrv_fetch($stmt)) {
fatalError("Failed to fetch row ");
}
$stream = sqlsrv_get_field($stmt, 0, SQLSRV_PHPTYPE_STREAM(SQLSRV_ENC_CHAR));
$success = false;
if ($stream !== false) {
$value = '';
$num = 0;
while (!feof($stream)) {
$value .= fread($stream, 8192);
$res = sqlsrv_execute($stmt);
if (!$res) {
fatalError("Failed to insert data");
}
fclose($stream);
if (checkData($value, $inValue)) { // compare the data to see if they match!
$success = true;
}
}
if ($success) {
echo "Done.\n";
} else {
fatalError("Failed to fetch stream ");
fatalError("Failed to prepare insert statement");
}
runTest($conn, false);
runTest($conn, true);
dropTable($conn, $tableName);
sqlsrv_free_stmt($stmt);
sqlsrv_close($conn);
echo "Done\n";
///////////////////////////////////////////////////////////////////////////////////////////////
function runTest($conn, $buffered)
{
global $tableName, $strValue, $nstrValue;
trace("runTest ($buffered)\n");
$query = "SELECT * FROM $tableName";
if ($buffered) {
$stmt = sqlsrv_prepare($conn, $query, null, array("Scrollable"=>SQLSRV_CURSOR_CLIENT_BUFFERED));
} else {
$stmt = sqlsrv_prepare($conn, $query);
}
if (!$stmt) {
fatalError("runTest ($buffered): failed to prepare select statement");
}
if (!sqlsrv_execute($stmt)) {
fatalError("runTest ($buffered): failed to execute select");
}
if (!sqlsrv_fetch($stmt)) {
fatalError("runTest ($buffered): failed to fetch data");
}
fetchAsString($stmt, 0, $strValue);
fetchAsString($stmt, 1, $nstrValue);
if (!sqlsrv_execute($stmt)) {
fatalError("runTest ($buffered): failed to execute select");
}
if (!sqlsrv_fetch($stmt)) {
fatalError("runTest ($buffered): failed to fetch data");
}
fetchAsStream($stmt, 0, $strValue);
fetchAsStream($stmt, 1, $nstrValue);
}
function fetchAsString($stmt, $index, $expected)
{
trace("fetchAsString ($index):\n");
$sqltype = ($index > 0) ? SQLSRV_PHPTYPE_STRING('UTF-8') : SQLSRV_PHPTYPE_STRING(SQLSRV_ENC_CHAR);
$value = sqlsrv_get_field($stmt, $index, $sqltype);
if (!checkData($value, $expected)) {
echo("fetchAsString ($index) expected:\n$expected\nActual:\n$value\n");
}
}
function fetchAsStream($stmt, $index, $expected)
{
trace("fetchAsStream ($index):\n");
$sqltype = ($index > 0) ? SQLSRV_PHPTYPE_STREAM('UTF-8') : SQLSRV_PHPTYPE_STREAM(SQLSRV_ENC_CHAR);
$stream = sqlsrv_get_field($stmt, $index, $sqltype);
if ($stream !== false) {
$value = '';
while (!feof($stream)) {
$value .= fread($stream, 8192);
}
fclose($stream);
if (!checkData($value, $expected)) {
echo("fetchAsStream ($index) expected:\n$expected\nActual:\n$value\n");
}
}
}
function checkData($actual, $expected)
@ -58,15 +116,11 @@ function checkData($actual, $expected)
$pos = strpos($actual, $expected);
if (($pos === false) || ($pos > 1)) {
$success = false;
$success = false;
}
if (!$success) {
trace("\nData error\nExpected:\n$expected\nActual:\n$actual\n");
}
return ($success);
}
?>
--EXPECT--
Done.
Done

View file

@ -0,0 +1,202 @@
--TEST--
Test fetching varbinary max fields with client buffer
--DESCRIPTION--
Test fetching varbinary max fields as streams or strings using client buffer
--SKIPIF--
<?php require('skipif_versions_old.inc'); ?>
--ENV--
PHPT_EXEC=true
--FILE--
<?php
require_once('MsCommon.inc');
function fetchNull($stmt, $sqltype, $message)
{
$result = sqlsrv_get_field($stmt, 1, $sqltype);
if (!is_null($result)) {
echo("$message: expected NULL\n");
}
}
function fetchNullStream($stmt, $sqltype, $message)
{
$stream = sqlsrv_get_field($stmt, 1, $sqltype);
if ($stream !== false) {
$value = fread($stream, 8192);
fclose($stream);
if (!empty($value)) {
echo("$message: expected an empty value\n");
}
}
}
function fetchStream($stmt, $test)
{
global $binaryValue, $hexValue;
if (!sqlsrv_execute($stmt)) {
fatalError("fetchStream: failed to execute select");
}
if (!sqlsrv_fetch($stmt)) {
fatalError("fetchStream: failed to fetch row");
}
switch ($test) {
case 1:
$sqltype = SQLSRV_PHPTYPE_STREAM(SQLSRV_ENC_CHAR);
$type = 'char string';
$expected = $hexValue;
break;
case 2:
$sqltype = SQLSRV_PHPTYPE_STREAM('UTF-8');
$type = 'UTF-8 string';
$expected = $hexValue;
break;
case 3:
$sqltype = SQLSRV_PHPTYPE_STREAM(SQLSRV_ENC_BINARY);
$type = 'binary string';
$expected = $binaryValue;
break;
default:
echo "fetchStream: something went wrong\n";
break;
}
trace("fetchStream ($type):\n");
$stream = sqlsrv_get_field($stmt, 0, $sqltype);
if ($stream !== false) {
$value = '';
while (!feof($stream)) {
$value .= fread($stream, 8192);
}
fclose($stream);
if (!checkData($value, $expected)) {
echo("fetchStream ($type)\nExpected:\n$expected\nActual:\n$value\n");
}
} else {
fatalError("fetchStream ($type) failed");
}
fetchNullStream($stmt, $sqltype, "fetchStream ($type)\n");
}
function fetchData($stmt, $test)
{
global $binaryValue, $hexValue;
if (!sqlsrv_execute($stmt)) {
fatalError("fetchData: failed to execute select");
}
if (!sqlsrv_fetch($stmt)) {
fatalError("fetchData: failed to fetch row");
}
switch ($test) {
case 1:
$sqltype = SQLSRV_PHPTYPE_STRING(SQLSRV_ENC_CHAR);
$type = 'char string';
$expected = $hexValue;
break;
case 2:
$sqltype = SQLSRV_PHPTYPE_STRING('UTF-8');
$type = 'UTF-8 string';
$expected = $hexValue;
break;
case 3:
$sqltype = SQLSRV_PHPTYPE_STRING(SQLSRV_ENC_BINARY);
$type = 'binary string';
$expected = $binaryValue;
break;
default:
echo "fetchData: something went wrong\n";
break;
}
trace("fetchData ($type):\n");
$value = sqlsrv_get_field($stmt, 0, $sqltype);
if (!checkData($value, $expected)) {
echo("fetchData ($type)\nExpected:\n$expected\nActual:\n$value\n");
}
fetchNull($stmt, $sqltype, "fetchData ($type)\n");
}
function runTest($conn, $buffered)
{
global $tableName, $binaryValue, $hexValue;
$query = "SELECT * FROM $tableName";
if ($buffered) {
trace("Test using a client buffer\n");
$stmt = sqlsrv_prepare($conn, $query, null, array("Scrollable"=>SQLSRV_CURSOR_CLIENT_BUFFERED));
} else {
trace("Test without using a client buffer\n");
$stmt = sqlsrv_prepare($conn, $query);
}
if (!$stmt) {
fatalError("runTest: failed to prepare select statement");
}
fetchData($stmt, 1);
fetchData($stmt, 2);
fetchData($stmt, 3);
fetchStream($stmt, 1);
fetchStream($stmt, 2);
fetchStream($stmt, 3);
}
function checkData($actual, $expected)
{
$success = true;
$pos = strpos($actual, $expected);
if (($pos === false) || ($pos > 1)) {
$success = false;
}
return ($success);
}
$conn = AE\connect();
$tableName = "binary_max_fields";
$columns = array(new AE\ColumnMeta("varbinary(max)", "varbinary_max_col"),
new AE\ColumnMeta("varbinary(max)", "varbinary_null_col"));
AE\createTable($conn, $tableName, $columns);
$bin = 'abcdefghijk';
$binaryValue = str_repeat($bin, 400);
$hexValue = strtoupper(bin2hex($binaryValue));
$insertSql = "INSERT INTO $tableName (varbinary_max_col, varbinary_null_col) VALUES (?, ?)";
$params = array(array($binaryValue, null, SQLSRV_PHPTYPE_STRING(SQLSRV_ENC_BINARY), SQLSRV_SQLTYPE_VARBINARY('max')),
array(null, null, SQLSRV_PHPTYPE_STRING(SQLSRV_ENC_CHAR), SQLSRV_SQLTYPE_VARBINARY('max')));
$stmt = sqlsrv_prepare($conn, $insertSql, $params);
if ($stmt) {
$res = sqlsrv_execute($stmt);
if (!$res) {
fatalError("Failed to insert data");
}
} else {
fatalError("Failed to prepare insert statement");
}
runTest($conn, false);
runTest($conn, true);
echo "Done\n";
dropTable($conn, $tableName);
sqlsrv_free_stmt($stmt);
sqlsrv_close($conn);
?>
--EXPECT--
Done