Skip to main content
added 2772 characters in body; edited tags; edited title
Source Link
b3m2a1
  • 47.1k
  • 3
  • 95
  • 245

Find Import/Export format data

Added export scraper

Here's a scraper for export formats:

getFormatExportDataRaw[fmt_] := 
  Block[{$Context = 
     "ItsASecretToEverybody`Private`", $ContextPath = {"System`"}}, 
   Replace[
    SelectFirst[
     Map[Function@FileNameJoin@{#, fmt, "Export.m"}, 
      System`ConvertersDump`$FormatsDirectory],
     FileExistsQ, $Failed], 
    s_String :> s -> Import[s, "HeldExpressions"]
    ]
   ];
getFormatExportData[fmt_] :=
 Module[{data, file},
  data = getFormatExportDataRaw[fmt];
  If[data =!= $Failed,
   {file, data} = List @@ data;
   data /. {
     HoldPattern[Set[s_Symbol, v : _List | _Association]] :>
      RuleCondition[Set[s, v], True]
     };
   With[{cont = 
      FirstCase[data, HoldPattern[Begin[c_]] :> c, 
       "System`FakeHo`", \[Infinity]]},
    SortBy[
       Switch[#[[1]], "FormatName", 0, "BinaryFormat", 2, "File", 
         3, _, 1] &
       ]@
      Flatten@{
        "DocumentedElements" ->
         FirstCase[data,
          HoldPattern[
            Set[
             _Symbol?(
               Function[Null,
                SymbolName[Unevaluated@#] == "$DocumentedElements",
                HoldFirst
                ]),
             v_]
            ] :> Hold@v, None, \[Infinity]],
        "HiddenElements" ->
         FirstCase[data,
          HoldPattern[
            Set[
             _Symbol?(
               Function[Null,
                SymbolName[Unevaluated@#] == "$HiddenElements",
                HoldFirst
                ]),
             v_
             ]
            ] :> Hold@v,
          None,
          \[Infinity]],
        FirstCase[data,
         HoldPattern[
           ImportExport`RegisterExport[fmtName_,
            function_,
            ruls___?System`ConvertersDump`RuleQ
            ]
           ] :> {
           "FormatName" -> fmtName,
           "Function" -> function,
           Replace[{ruls},
            ("AvailableElements" -> _Symbol) -> Nothing,
            1],
           "File" -> file
           },
         None,
         \[Infinity]]
        } /. s_Symbol?(
        Function[Null,
         Context[#] == "ItsASecretToEverybody`Private`",
         HoldFirst
         ]
        ) :>
      RuleCondition[
       ToExpression[cont <> SymbolName[Unevaluated[s]],
        StandardForm,
        Inactivate
        ],
       True]
    ],
   {}
   ]
  ]

Find Import format data

Find Import/Export format data

Added export scraper

Here's a scraper for export formats:

getFormatExportDataRaw[fmt_] := 
  Block[{$Context = 
     "ItsASecretToEverybody`Private`", $ContextPath = {"System`"}}, 
   Replace[
    SelectFirst[
     Map[Function@FileNameJoin@{#, fmt, "Export.m"}, 
      System`ConvertersDump`$FormatsDirectory],
     FileExistsQ, $Failed], 
    s_String :> s -> Import[s, "HeldExpressions"]
    ]
   ];
getFormatExportData[fmt_] :=
 Module[{data, file},
  data = getFormatExportDataRaw[fmt];
  If[data =!= $Failed,
   {file, data} = List @@ data;
   data /. {
     HoldPattern[Set[s_Symbol, v : _List | _Association]] :>
      RuleCondition[Set[s, v], True]
     };
   With[{cont = 
      FirstCase[data, HoldPattern[Begin[c_]] :> c, 
       "System`FakeHo`", \[Infinity]]},
    SortBy[
       Switch[#[[1]], "FormatName", 0, "BinaryFormat", 2, "File", 
         3, _, 1] &
       ]@
      Flatten@{
        "DocumentedElements" ->
         FirstCase[data,
          HoldPattern[
            Set[
             _Symbol?(
               Function[Null,
                SymbolName[Unevaluated@#] == "$DocumentedElements",
                HoldFirst
                ]),
             v_]
            ] :> Hold@v, None, \[Infinity]],
        "HiddenElements" ->
         FirstCase[data,
          HoldPattern[
            Set[
             _Symbol?(
               Function[Null,
                SymbolName[Unevaluated@#] == "$HiddenElements",
                HoldFirst
                ]),
             v_
             ]
            ] :> Hold@v,
          None,
          \[Infinity]],
        FirstCase[data,
         HoldPattern[
           ImportExport`RegisterExport[fmtName_,
            function_,
            ruls___?System`ConvertersDump`RuleQ
            ]
           ] :> {
           "FormatName" -> fmtName,
           "Function" -> function,
           Replace[{ruls},
            ("AvailableElements" -> _Symbol) -> Nothing,
            1],
           "File" -> file
           },
         None,
         \[Infinity]]
        } /. s_Symbol?(
        Function[Null,
         Context[#] == "ItsASecretToEverybody`Private`",
         HoldFirst
         ]
        ) :>
      RuleCondition[
       ToExpression[cont <> SymbolName[Unevaluated[s]],
        StandardForm,
        Inactivate
        ],
       True]
    ],
   {}
   ]
  ]
Tweeted twitter.com/StackMma/status/954756501889081344
Source Link
b3m2a1
  • 47.1k
  • 3
  • 95
  • 245

Find Import format data

Often Import related questions come up, for which I'm always wanting to inspect the proper import function used to see what Options and features I can get out of it.

For this I generally use this function:

findImportFormatFile[fmt_] :=
 SelectFirst[
  Map[Function @ FileNameJoin @ {#, fmt, "Import.m"},
   System`ConvertersDump`$FormatsDirectory
   ],
  FileExistsQ,
  $Failed
  ]

which for instance can find the Import definitions for "CSV":

findImportFormatFile["CSV"]

"/Applications/Mathematica.app/Contents/SystemFiles/Formats/CSV/Import.m"

from which we can glean that the base import function is:

System`Convert`TableDump`ImportSV

but in a fresh kernel this isn't defined, even after we use System`ConvertersDump`LoadImportFormat.

Looking at the data further we see the issue is the "Sources" for the importer haven't been loaded:

ImportExport`DefaultSources["Table"]

{"/Applications/Mathematica.app/Contents/SystemFiles/Kernel/SystemResources/MacOSX-x86-64/Convert/Table.mx"}

And after we import this we're good.

But I'd like a way to get all this data--import function, options names, "Sources", etc.--automatically.

How can I do this?


Helper code

Here's a partial solution I wrote up before thinking to pose this question. Hopefully it's helpful if someone wants to roll a scrape, if there's no more legit way to do this.

getFormatImportDataRaw[fmt_] :=
  
  Block[{$Context = 
     "ItsASecretToEverybody`Private`", $ContextPath = {"System`"}},
   Replace[
    SelectFirst[
     Map[Function @ FileNameJoin @ {#, fmt, "Import.m"},
      System`ConvertersDump`$FormatsDirectory
      ],
     FileExistsQ,
     $Failed
     ],
    s_String :> s -> Import[s, "HeldExpressions"]
    ]
   ];
getFormatImportData[fmt_] :=
 Module[{
   data, 
   file
   },
  data = getFormatImportDataRaw[fmt];
  If[data =!= $Failed,
   {file, data} = List @@ data;
   data /. {
     HoldPattern[Set[s_Symbol, v : _List | _Association]] :>
      RuleCondition[Set[s, v], True]
     };
   With[{cont = 
      FirstCase[data, HoldPattern[Begin[c_]] :> c, 
       "System`FakeHo`", ∞]},
    SortBy[
       Switch[#[[1]], 
         "FormatName", 0,
         "BinaryFormat", 2,
         "File", 3,
         _, 1
         ] &
       ]@
      Flatten@{
        "DocumentedElements" ->
         FirstCase[data, 
          HoldPattern[
            Set[_Symbol?(Function[Null, 
                SymbolName[Unevaluated@#] == "$DocumentedElements", 
                HoldFirst]), v_]] :> Hold@v,
          None,
          ∞
          ],
        "HiddenElements" ->
         FirstCase[data, 
          HoldPattern[
            Set[_Symbol?(Function[Null, 
                SymbolName[Unevaluated@#] == "$HiddenElements", 
                HoldFirst]), v_]] :> Hold@v,
          None,
          ∞
          ],
        FirstCase[data,
         HoldPattern[
           ImportExport`RegisterImport[
            fmtName_, 
            raw_,
            post : _List : {},
            ruls___?System`ConvertersDump`RuleQ
            ]
           ] :>
          {
           "FormatName" -> fmtName,
           Replace[
            Flatten@List@raw,
            {
             (r : Rule | RuleDelayed)[k_, h_] :>
              r[k, h],
             f : Except[_Rule | _RuleDelayed] :> "DefaultImport" -> f
             },
            1
            ],
           Replace[
            post,
            {
             (r : Rule | RuleDelayed)[k_, h_] :>
              r[k, h],
             f : Except[_Rule | _RuleDelayed] :> "DefaultImport" -> f
             },
            1
            ],
           Replace[{ruls},
            ("AvailableElements" -> _Symbol) -> Nothing,
            1
            ],
           "File" -> file
           },
         None,
         ∞
         ]
        } /. 
     s_Symbol?(Function[Null, 
         Context[#] == "ItsASecretToEverybody`Private`", 
         HoldFirst]) :> 
      RuleCondition[
       ToExpression[cont <> SymbolName[Unevaluated[s]], StandardForm, 
        Inactivate], True]
    ],
   {}
   ]
  ]

It's not great and if there's a built-in way I'd much, much prefer that.

It does work for some formats, though:

getFormatImportData@"CSV" // 
  Take[#, 3] & // GeneralUtilities`PrettyForm

{
    "FormatName" -> "CSV", 
    "DefaultElement" -> "Data", 
    "DefaultImport" -> (System`Convert`TableDump`ImportSV["CSV", System`Convert`TableDump`elem, All, SlotSequence[1]] &)
 }