Output Clause Adds Extra Characters Not In Originating source

Question

After an insert into a temp table using the OUTPUT clause, I am getting an extra character(s) that are not in the source.

Why?

--

E.g. after insertion from existing PhysicalTable_1 table record with LName = 'John' the destination PhysicalTable_1 table as well as the #Temp table have '?John' or 'I?John' it occurs sometime for Lname, sometime for FName or Email as well as other fields.

An example of data in the PhysicalTable_1 - FName = '​Raul' And destination record after insertion looks like = '?Raul'

I'm using this:

 CREATE TABLE #Temp 
    (
    ID INT NOT NULL,
    LName VARCHAR(75) NULL,
    FName VARCHAR(75) NULL,
    Email VARCHAR(125) NULL
    )
    
    CREATE TABLE PhysicalTable_2
    (
    ID INT NOT NULL,
    LName VARCHAR(75) NULL,
    FName VARCHAR(75) NULL,
    Email VARCHAR(125) NUL
    )
    
 CREATE TABLE PhysicalTable_1
    (
    ID INT NOT NULL,
    LName NVARCHAR(500) NULL,
    FName NVARCHAR(500) NULL,
    Email NVARCHAR(500) NULL
    )

    INSERT INTO PhysicalTable_2
    (
      LName, FName, Email
    )
    OUTPUT INSERTED.LName, INSERTED.FName, INSERTED.Email
    INTO #Temp
    
    SELECT LName, FName, Email
    FROM PhysicalTable_1

I also tried to change all string fields data types of #Temp table to NVARCHAR. Still some records in the destination ended up having extra characters

Answer 1

The problem is that your PhysicalTable_1 contains non-printable unicode characters in LName. You insert the unicode LName NVARCHAR column of Table1, into an ascii/nonunicode LName VARCHAR column of Table2. Nonunicode is half the size of unicode in sql server, some bytes have to be "cut-off" and because of the size reduction the non-printable characters become apparent.

--characters to binary
SELECT CAST(N'P' AS VARBINARY(10)) AS UnicodeP, CAST('P' AS VARBINARY(10)) AS AsciiP --unicode is double the size of ascii


CREATE TABLE #temp(UnicodeP NVARCHAR(10), AsciiP VARCHAR(10));

INSERT INTO #temp(UnicodeP, AsciiP) VALUES (N'P', 'P'); --nothing special, normal insertion
INSERT INTO #temp(UnicodeP, AsciiP) VALUES ('P', N'P'); --omitting the N for unicode and using N for ascii, still works ok, implicit conversion

SELECT * FROM #temp;

--use binary from the very first SELECT CAST(....
INSERT INTO #temp(UnicodeP, AsciiP) VALUES (0x5000, 0x50); --still fine
SELECT * FROM #temp;

--prepend a nonprintable character (BOM) to unicode P, just insert into the UnicodeP only
INSERT INTO #temp(UnicodeP, AsciiP) VALUES (0xFEFF + 0x5000, NULL); --still fine
SELECT * FROM #temp;

--if you copy and paste the last UnicodeP, where AsciiP is NULL, you will not notice any visual difference


--update the ascii from unicode , where ascii is null
UPDATE #temp
SET AsciiP = UnicodeP --implicit conversion, ascii is half the unicode, some bytes have to go away
WHERE AsciiP IS NULL;

--since unicode was implicitly converted to ascii,  some bytes are "stripped out"  The nonprintable 0xFEFF needs to be "cut in half" and it becomes an unidentified char
SELECT UnicodeP, CAST(UnicodeP AS VARBINARY(10)) AS UnicodePbinary, AsciiP, CAST(AsciiP AS VARBINARY(10)) as AsciiPbinary
FROM #temp;


DROP TABLE #temp;

*edit, implicit unicode to nonunicode and asciiOrnothing

SELECT NCHAR(rownum) AS TheChar, CAST(NCHAR(rownum) AS CHAR(1)) AS ImplicitConversion, 
    CASE WHEN NCHAR(rownum) < N'Ā' collate Latin1_General_BIN2 THEN NCHAR(rownum) ELSE '' END AS AsciiOrNothing,
    UNICODE(NCHAR(rownum)) AS CharInteger,
    --or
    CASE WHEN UNICODE(/*TheChar*/ NCHAR(rownum)) <= 255 THEN NCHAR(rownum) ELSE '' END AS AsciiOrNothing2
FROM 
(
SELECT ROW_NUMBER() OVER(ORDER BY (SELECT null)) AS rownum
FROM (
    --10K
    SELECT TOP (100) name from master.dbo.spt_values) AS a
    CROSS JOIN (SELECT TOP (100) name from master.dbo.spt_values) AS b
) AS src
ORDER BY rownum