Operation JavaScript Transformation

About me

@BrianDiPalma1

https://github.com/briandipalma/global-compiler

https://github.com/briandipalma/gc-cli

A little history...

We started writing large JS web apps 8+ years ago.

jQuery was in its infancy.

Before AMD.

Before CJS.

Before ES6 modules.

...so we copied what we knew

And we knew Java.

novox.fxtrader.grid.decorator.MyDecorator = function(mDecoratorConfig)
{
	this.clickedClass = mDecoratorConfig.clickedClass;
};

caplin.implement(novox.fxtrader.grid.decorator.MyDecorator, caplin.grid.GridViewListener);

// GridDecorator Interface Methods
novox.fxtrader.grid.decorator.MyDecorator.prototype.setGridView = function(oGridView)
{
   // register the decorator as a listener	
   this.m_oGridView = oGridView;
   oGridView.addGridViewListener(this);
};

novox.fxtrader.grid.decorator.MyDecorator.prototype._onClick = function(oEvent)
{
   oEvent = caplin.dom.event.Event.getNormalizedEvent(oEvent);
   var eRow = caplin.dom.Utility.getAncestorElementWithClass(oEvent.target, "row");

   if(caplin.dom.Utility.hasClassName(eRow, this.clickClass)) {   
      caplin.dom.Utility.removeClassName(eRow, this.clickClass);
   } else {
      caplin.dom.Utility.addClassName(eRow, this.clickClass);
   }
};

Custom Loader

Code was loaded by a program that scanned the source code for strings that matched known class names.

If it found a match in a file it would include the matched file.

At that point all was good...

A wild MODULE appeared!

Eventually the JS community solved the code structure and loading problem too.

They devised AMD and CJS.

These were used as inspiration for the ES6 module specification.

Modules are the future

That combined with the readability improvements of using CJS convinced us to add support for CJS.

The plan was to use CJS for new code while allowing us to still use our current code.

No one likes legacy code

Maybe we could make all the code look like the shiny and new code?

Giving us more readable code and consistent style and mental model.

Could we convert the old style code?

Approaches

• Manual conversion
• Regular expressions
• Find and replace
• Parse and mutate the code

Approaches

• Manual conversion Laborious, repetitive work is error prone, can't justify cost/time
• Regular expressions Error prone, inflexible, complexity of edge cases.
• Find and replace Error prone, inflexible, complexity of edge cases.
• Parse and mutate the code

novox.fxtrader.grid.decorator.MyDecorator = function(mDecoratorConfig)
{
	this.clickedClass = mDecoratorConfig.clickedClass;
};

caplin.implement(novox.fxtrader.grid.decorator.MyDecorator, caplin.grid.GridViewListener);

// GridDecorator Interface Methods
novox.fxtrader.grid.decorator.MyDecorator.prototype.setGridView = function(oGridView)
{
   // register the decorator as a listener	
   this.m_oGridView = oGridView;
   oGridView.addGridViewListener(this);
};

novox.fxtrader.grid.decorator.MyDecorator.prototype._onClick = function(oEvent)
{
   oEvent = caplin.dom.event.Event.getNormalizedEvent(oEvent);
   var eRow = caplin.dom.Utility.getAncestorElementWithClass(oEvent.target, "row");

   if(caplin.dom.Utility.hasClassName(eRow, this.clickClass)) {   
      caplin.dom.Utility.removeClassName(eRow, this.clickClass);
   } else {
      caplin.dom.Utility.addClassName(eRow, this.clickClass);
   }
};

Convert this

var topiarist = require('topiarist');
var Utility = require('caplin/dom/Utility');
var Event = require('caplin/dom/event/Event');
var GridViewListener = require('caplin/grid/GridViewListener');

function MyDecorator(mDecoratorConfig) {
    this.clickedClass = mDecoratorConfig.clickedClass;
};

topiarist.implement(MyDecorator, GridViewListener);

// GridDecorator Interface Methods
MyDecorator.prototype.setGridView = function(oGridView) {
   // register the decorator as a listener	
   this.m_oGridView = oGridView;
   oGridView.addGridViewListener(this);
};

MyDecorator.prototype._onClick = function(oEvent) {
   oEvent = Event.getNormalizedEvent(oEvent);
   var eRow = Utility.getAncestorElementWithClass(oEvent.target, "row");

   if(Utility.hasClassName(eRow, this.clickClass)) {   
      Utility.removeClassName(eRow, this.clickClass);
   } else {
      Utility.addClassName(eRow, this.clickClass);
   }
};

module.exports = MyDecorator;

to this

Parsers and ASTs

• Esprima
• Acorn
• Espree
• Babylon

They all output ESTree compliant ASTs

var ast = esprima.parse('var answer = 42', options);

ASTs

Data structure representing abstract syntactic code structure.

Doesn't keep track of data that has no

execution semantics.

So code like this...

my.long.name.space.Field

is represented like...

"type": "MemberExpression",
"computed": false,
"object": {
    "type": "MemberExpression",
    "computed": false,
    "object": {
        "type": "MemberExpression",
        "computed": false,
        "object": {
            "type": "MemberExpression",
            "computed": false,
            "object": {
                "type": "Identifier",
                "name": "my"
            },
            "property": {
                "type": "Identifier",
                "name": "long"
            }
        },
        "property": {
            "type": "Identifier",
            "name": "name"
        }
    },
    "property": {
        "type": "Identifier",
        "name": "space"
    }
},
"property": {
    "type": "Identifier",
    "name": "Field"
}

Identifier

An identifier. Note that an identifier may be an expression or a destructuring pattern.

interface Identifier <: Node, Expression, Pattern {
    type: "Identifier";
    name: string;
}

Node types

http://esprima.org/demo/parse.html

http://eslint.org/parser/

http://astexplorer.net/

Transforming ASTs

Recast

https://github.com/benjamn/recast

import {parse, types, print} from 'recast';

// Parse the code using an interface similar to esprima.parse.
const ast = parse('my.long.name.space.Field');

// Recast allows you to safely build new AST nodes.
const moduleIdentifier = types.builders.identifier('Field');

// You can modify the AST by replacing the "ExpressionStatement" node
// "expression" property with your newly created "Identifier".
ast.program.body[0].expression = moduleIdentifier;

// Field
print(ast).code;

import {visit} from 'recast';

...

const flatteningVisitor = {
  visitIdentifier(path) {
    // do something with path
    this.traverse(path);
  },

  visitExpressionStatement(path) {
    // do something with path
    this.traverse(path);
  }
};

visit(ast, flatteningVisitor);

Traversing ASTs

You need to find the nodes to transform.

Visitor pattern is the standard approach.

...

visitIdentifier(identifierNodePath) {
    const {parent} = identifierNodePath;

    // Is this identifier the leaf a fully qualified name.
    if (isClassNamespaceLeaf(identifierNodePath, parent, this._namespaceList)) {
        // Recast allows you to safely build new AST nodes.
        const classIdentifier = types.builders.identifier(this._className);

        // It allows you to replace AST nodes.
        identifierNodePath.replace(classIdentifier);
    }

    this.traverse(path);
},

...

NodePaths

What Recast provides to the visitor callbacks.

Provide API to mutate underlying AST e.g. replace, prune, unshift, shift, etc.

Also provides scope information e.g. isGlobal, declares

// Matcher that matches `caplin.extend()` or `caplin.implement()`
const caplinInheritanceMatcher = composeMatchers(
	identifierMatcher('caplin'),
	orMatchers(
		memberExpressionMatcher({property: identifierMatcher('extend')}),
		memberExpressionMatcher({property: identifierMatcher('implement')})
	),
	callExpressionMatcher()
);

// Transformer that converts caplin.extend/implement to topiarist.extend
const caplinInheritanceToExtendTransformer = composeTransformers(
	identifier('topiarist'),
	extractParent(),
	extractProperties('property'),
	identifier('extend')
);

// Matchers and transforms are passed into a generic visitor

Functional composition

Some transforms can be described as static matcher and transform pairs.

Multiple Transforms

export function compileSourceFiles(options) {
    vinylFs.src('src/**/*.js')
        .pipe(parseJSFile())
        .pipe(expandVarNamespaceAliases(options.namespaces))
        .pipe(flattenIIFEClass())
        .pipe(flattenClass())
        .pipe(convertGlobalsToRequires(options.namespaces))
        .pipe(addRequiresForLibraries(options.libraryIdentifiersToRequire))
        .pipe(transformI18nUsage())
        .pipe(convertASTToBuffer())
        .pipe(vinylFs.dest(options.outputDirectory))
        .on('end', createJSStyleFiles());
}

Keep your transforms as small and focused as you can. Helps deal with myriad of code patterns in real world.

Make your transforms configurable.

export const namespacedClassFlattenerVisitor = {
	/**
	 * @param {string} fullyQualifiedName The fully qualified class name
	 */
	initialize(fullyQualifiedName) {
		const nameParts = fullyQualifiedName.split('.').reverse();

		this._namespaceList = List.of(...nameParts);
		this._className = this._namespaceList.first();
	}

...

jscodeshift

For one-off, stand alone transforms.

$ jscodeshift --transform myTransform.js src/.

/**
 * This replaces every occurence of variable "foo".
 */
module.exports = function(fileInfo, api) {
  return api.jscodeshift(fileInfo.source)
    .findVariableDeclarators('foo')
    .renameTo('bar')
    .toSource();
}

Provides access to recast builders.

Supports stats/dry runs and template literals.

js-codemod has sample transforms.

// Searching for all Identifiers.
jscodeshift(input)
        .find(jscodeshift.Identifier)
        .forEach(function(path) {
            // do something with path
         });

// How to use Template Literals.
jscodeshift(input)
        .find('ForStatement')
        .replaceWith(
          ({node}) => statements`
            ${node.init};
            while (${node.test}) {
              ${node.body.body}
              ${node.update}
            }`
        )
        .toSource();

examples of the jscodeshift API

$ jscodeshift -t myTransforms fileA fileB --foo=bar

Also supports passing in CLI options.

Babel Plugin

export default function ({ Plugin, types: t }) {
  return new Plugin("foo-bar", {
    visitor: {
      FunctionDeclaration(node, parent, scope) {
        this.parentPath.replaceWith(
          t.expressionStatement(t.literal("no function declarations allowed!"))
        );

        if (scope.hasOwnBinding("name")) {
            // "name" bound in this scope.
        }
      }
    }
  });
}

Useful for build time, reusable transforms.

Rich API e.g. replace with multiple nodes, with source string,

rename bindings, visitor aliases (Function for FunctionDeclaration/Expression), unique identifier creation.

Enough with the Abstract!

What about style guides?

Linter rules?

Recast can have its output configured but it's not very flexible.

esformatter

Highly configurable for indentation and white space

{
  "indent": {
    "value": "\t",
    "FunctionExpression": 1,
    "ArrayExpression": 0,
    "ObjectExpression": 0
  }
}

Plugins can be used for other types formatting changes.

• esformatter-quotes
• esformatter-spaced-lined-comment
• esformatter-quote-props

module.exports.nodeBefore = function(node) {
	if (isQuotedProperty(node) && isSafeToUnquote(node)) {
		unquoteProperty(node);
	}
};

Operate on source string, tokens or on AST nodes

• setOptions
• stringBefore 
• transformBefore
• tokenBefore
• nodeBefore
• nodeAfter 
• tokenAfter
• transformAfter
• stringAfter

module.exports.tokenBefore = function(token) {
	if (token.type === 'LineComment' && token.value.match(/^\S/)) {
		token.raw = '// ' + token.value;
	}
};

Congratulations!

You have transformed an old code base into a modern code base that follows your code style guide, passes linting checks.

Don't be afraid to contemplate huge changes to your code base.

Questions?