VSX: How can I reuse the existing XML editor to handle binary files converted to XML?

Question

I'm trying to create an extension VSPackage for VS2017 (in C#) which would convert binary data to XML, opens that in the default VS XML editor and XML language service, and then converts it back to binary upon saving.

However, I have troubles to line out which steps would be required for this. I thought of the following for now when creating a new editor in the editor factory:

• Create new text buffer
• Feed it with converted XML data
• Create core editor
• Feed it with the text buffer

Right now my attempt looks like this:

private MyPackage _package; // Filled via constructor
private IServiceProvider _serviceProvider; // Filled via SetSite

public int CreateEditorInstance(uint grfCreateDoc, string pszMkDocument, string pszPhysicalView,
    IVsHierarchy pvHier, uint itemid, IntPtr punkDocDataExisting, out IntPtr ppunkDocView,
    out IntPtr ppunkDocData, out string pbstrEditorCaption, out Guid pguidCmdUI, out int pgrfCDW)
{
    // Initialize and validate parameters.
    ppunkDocView = IntPtr.Zero;
    ppunkDocData = IntPtr.Zero;
    pbstrEditorCaption = String.Empty;
    pguidCmdUI = Guid.Empty;
    pgrfCDW = 0;
    VSConstants.CEF createDocFlags = (VSConstants.CEF)grfCreateDoc;
    if (!createDocFlags.HasFlag(VSConstants.CEF.OpenFile) && !createDocFlags.HasFlag(VSConstants.CEF.Silent))
        return VSConstants.E_INVALIDARG;
    if (punkDocDataExisting != IntPtr.Zero)
        return VSConstants.VS_E_INCOMPATIBLEDOCDATA;

    // Create a sited IVsTextBuffer storing the converted data with the XML data and language service set.
    IVsTextLines textLines = _package.CreateComInstance<VsTextBufferClass, IVsTextLines>();
    SiteObject(textLines);
    string xmlText = BinaryXmlData.GetXmlString(pszMkDocument);
    textLines.InitializeContent(xmlText, xmlText.Length);
    ErrorHandler.ThrowOnFailure(textLines.SetLanguageServiceID(ref Guids.XmlLanguageServiceGuid));

    // Instantiate a sited IVsCodeWindow and feed it with the text buffer.
    IVsCodeWindow codeWindow = _package.CreateComInstance<VsCodeWindowClass, IVsCodeWindow>();
    SiteObject(codeWindow);
    codeWindow.SetBuffer(textLines);

    // Return the created instances to the caller.
    ppunkDocView = Marshal.GetIUnknownForObject(codeWindow);
    ppunkDocData = Marshal.GetIUnknownForObject(textLines);

    return VSConstants.S_OK;
}

private void SiteObject(object obj)
{
    (obj as IObjectWithSite)?.SetSite(_serviceProvider);
}

// --- CreateComInstance is a method on my package ----
internal TInterface CreateComInstance<TClass, TInterface>()
{
    Guid guidT = typeof(TClass).GUID;
    Guid guidInterface = typeof(TInterface).GUID;

    TInterface instance = (TInterface)CreateInstance(ref guidT, ref guidInterface, typeof(TInterface));
    if (instance == null)
        throw new COMException($"Could not instantiate {typeof(TClass).Name} / {typeof(TInterface).Name}.");

    return instance;
}

When I try to explicitly open a file with my editor, it states "The file cannot be opened with the selected editor. Please choose another editor." The message doesn't make sense to me, I tried to open XML data with the XML editor, but it somehow still tries to open a text editor with the binary data.

I'm stuck here, I did all I could think of to feed it converted data. Apparently this way is not the right one.

• How could I add steps inbetween to fetch the binary data, quickly convert it to XML, then feed it to the XML editor?
• How would I store it back as binary when the XML editor saves the file?
• Is it possible at all to reuse the XML editor and language services for this?

I'm sorry if these questions require lengthy answers; I'd be happy already if I could get pointed in the right direction or to some already open-sourced extension doing something similar (converting file data before displaying it in a VS code editor).

Answer 1

The general idea is to let the Xml Editor do what it usually does: open a document.

In your case, if I understand it correctly, you don't have a physical Xml document, so you have to create one. A document is something (it doesn't not have to be a physical file) that is registered in Visual Studio's Running Object Table.

Once you have a document you can just ask the shell to open it. You can use the ROT again to handle BeforeSave and AfterSave events. Here is some sample code that should do all this:

public int CreateEditorInstance(uint grfCreateDoc, string pszMkDocument, string pszPhysicalView, IVsHierarchy pvHier, uint itemid, IntPtr punkDocDataExisting, out IntPtr ppunkDocView, out IntPtr ppunkDocData, out string pbstrEditorCaption, out Guid pguidCmdUI, out int pgrfCDW)
{
    ppunkDocView = IntPtr.Zero;
    ppunkDocData = IntPtr.Zero;
    pbstrEditorCaption = null;
    pguidCmdUI = Guid.Empty;
    pgrfCDW = 0;

    // create your virtual Xml buffer
    var data = Package.CreateComInstance<VsTextBufferClass, IVsTextLines>();
    SiteObject(data);

    // this is where you're supposed to build your virtual Xml content from your binary data
    string myXml = "<root>blah</root>";
    data.InitializeContent(myXml, myXml.Length);
    var dataPtr = Marshal.GetIUnknownForObject(data);

    // build a document and register it in the Running Object Table
    // this document has no hierarchy (it will be handled by the 'Miscellaneous Files' fallback project)
    var rotFlags = _VSRDTFLAGS.RDT_ReadLock | _VSRDTFLAGS.RDT_VirtualDocument;

    // come up with a moniker (which will be used as the caption also by the Xml editor)
    // Note I presume the moniker is a file path, wich may not always be ok depending on your context
    var virtualMk = Path.ChangeExtension(pszMkDocument, ".xml");
    var rot = (IVsRunningDocumentTable)_sp.GetService(typeof(SVsRunningDocumentTable));
    int hr = rot.RegisterAndLockDocument((uint)rotFlags, virtualMk, null, VSConstants.VSITEMID_NIL, dataPtr, out uint docCookie);
    if (hr != 0)
        return hr;

    try
    {
        // ask Visual Studio to open that document
        var opener = (IVsUIShellOpenDocument)_sp.GetService(typeof(SVsUIShellOpenDocument));
        var view = VSConstants.LOGVIEWID_Primary;
        opener.OpenDocumentViaProject(virtualMk, ref view,
            out Microsoft.VisualStudio.OLE.Interop.IServiceProvider psp,
            out IVsUIHierarchy uiHier,
            out uint id,
            out IVsWindowFrame frame);
        if (frame != null)
        {
            // Hmm.. the dirty bit (the star after the caption) is not updated by the Xml Editor...
            // If you close the document (or close VS), it does update it, but it does not react when we type in the editor.
            // This is unexpected, so, let's do the "dirty" work ourselves
            // hook on text line events from the buffer
            var textLineEvents = new TextLineEvents((IConnectionPointContainer)data);

            // we want to know when to unadvise, to hook frame events too
            ((IVsWindowFrame2)frame).Advise(textLineEvents, out uint frameCookie);

            textLineEvents.LineTextChanged += (sender, e) =>
            {
                // get the dirty bit and override the frame's dirty state
                ((IVsPersistDocData)data).IsDocDataDirty(out int dirty);
                frame.SetProperty((int)__VSFPROPID2.VSFPROPID_OverrideDirtyState, dirty != 0 ? true : false);
            };

            // now handle save events using the rot
            var docEventHandler = new RotDocumentEvents(docCookie);
            docEventHandler.Saving += (sender, e) =>
            {
                // this is where you can get the content of the data and save your binary data back
                // you can use Saved or Saving

            };

            docEventHandler.Saved += (sender, e) =>
            {
                // manual reset of dirty bit...
                frame.SetProperty((int)__VSFPROPID2.VSFPROPID_OverrideDirtyState, false);
            };
            rot.AdviseRunningDocTableEvents(docEventHandler, out uint rootCookie);

            frame.Show();
        }
    }
    finally
    {
        rot.UnlockDocument((uint)_VSRDTFLAGS.RDT_ReadLock, docCookie);
    }
    return VSConstants.S_OK;
}

private class TextLineEvents : IVsTextLinesEvents, IVsWindowFrameNotify, IVsWindowFrameNotify2
{
    public event EventHandler LineTextChanged;
    private uint _cookie;
    private IConnectionPoint _cp;

    public TextLineEvents(IConnectionPointContainer cpc)
    {
        var textLineEventsGuid = typeof(IVsTextLinesEvents).GUID;
        cpc.FindConnectionPoint(ref textLineEventsGuid, out _cp);
        _cp.Advise(this, out _cookie);
    }

    public void OnChangeLineText(TextLineChange[] pTextLineChange, int fLast) => LineTextChanged?.Invoke(this, EventArgs.Empty);

    public int OnClose(ref uint pgrfSaveOptions)
    {
        _cp.Unadvise(_cookie);
        return VSConstants.S_OK;
    }

    public void OnChangeLineAttributes(int iFirstLine, int iLastLine) { }
    public int OnShow(int fShow) => VSConstants.S_OK;
    public int OnMove() => VSConstants.S_OK;
    public int OnSize() => VSConstants.S_OK;
    public int OnDockableChange(int fDockable) => VSConstants.S_OK;
}

private class RotDocumentEvents : IVsRunningDocTableEvents3
{
    public event EventHandler Saved;
    public event EventHandler Saving;

    public RotDocumentEvents(uint docCookie)
    {
        DocCookie = docCookie;
    }

    public uint DocCookie { get; }

    public int OnBeforeSave(uint docCookie)
    {
        if (docCookie == DocCookie)
        {
            Saving?.Invoke(this, EventArgs.Empty);
        }
        return VSConstants.S_OK;
    }

    public int OnAfterSave(uint docCookie)
    {
        if (docCookie == DocCookie)
        {
            Saved?.Invoke(this, EventArgs.Empty);
        }
        return VSConstants.S_OK;
    }

    public int OnAfterFirstDocumentLock(uint docCookie, uint dwRDTLockType, uint dwReadLocksRemaining, uint dwEditLocksRemaining) => VSConstants.S_OK;
    public int OnBeforeLastDocumentUnlock(uint docCookie, uint dwRDTLockType, uint dwReadLocksRemaining, uint dwEditLocksRemaining) => VSConstants.S_OK;
    public int OnAfterAttributeChange(uint docCookie, uint grfAttribs) => VSConstants.S_OK;
    public int OnBeforeDocumentWindowShow(uint docCookie, int fFirstShow, IVsWindowFrame pFrame) => VSConstants.S_OK;
    public int OnAfterDocumentWindowHide(uint docCookie, IVsWindowFrame pFrame) => VSConstants.S_OK;
    public int OnAfterAttributeChangeEx(uint docCookie, uint grfAttribs, IVsHierarchy pHierOld, uint itemidOld, string pszMkDocumentOld, IVsHierarchy pHierNew, uint itemidNew, string pszMkDocumentNew) => VSConstants.S_OK;
}