In build.go:

func (ctxt *Context) importGo(p *Package, path, srcDir string, mode ImportMode) error {
    // To invoke the go command,
    // we must not being doing special things like AllowBinary or IgnoreVendor,
    // and all the file system callbacks must be nil (we're meant to use the local file system).
    if mode&AllowBinary != 0 || mode&IgnoreVendor != 0 ||
        ctxt.JoinPath != nil || ctxt.SplitPathList != nil || ctxt.IsAbsPath != nil || ctxt.IsDir != nil || ctxt.HasSubdir != nil || ctxt.ReadDir != nil || ctxt.OpenFile != nil || !equal(ctxt.ToolTags, defaultToolTags) || !equal(ctxt.ReleaseTags, defaultReleaseTags) {
        return errNoModules
    }

ctxt.UseAllFiles == true is special and should be in the list.

/cc @filosottile