try_convert(..) is null

select * from table where field is not null and try_cast(field as xml) is null

create xml schema collection rootOnly
as '<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
    <xsd:element name="root">
		<xsd:complexType></xsd:complexType>
    </xsd:element>
</xsd:schema>'
go

declare @t table (val nvarchar(max));

insert into @t values ('<root></root>'), ('<root></root'), ('<root1></root1>'), ('<root><row></row></root>');

select * from @t where val is not null and try_cast(val as xml) is null;

select * from @t as t where t.val is not null and try_cast(val as xml(rootOnly)) is not null;

go

drop xml schema collection rootOnly;