How do I add custom validation to an editable report?

For ExtJS-based reports, you can use a report plugin which adds a custom validator to the grid's One.grid.GridErrorsPlugin instance. Here's a link to the documentation for the validator function.

define(function() {

Ext.ns('MOD');
MOD.ReportPlugin = {
  init: function (report) {
    report.on('gridcreated', function(_, grid) {
      report.gridErrors.validators.push(function(record) {
        // TODO
        
        return {
          message: 'Row-level error message'
          ,fields:[{
            name: 'Site$Name', message: 'Field-level error message'
          }]
        };
      });
    });
  }
};

});