Running Tests from IDE works but not from command line

Question

I've written Unit Tests, Instrumentation Tests and Espresso Tests. I also run them with Android Test Orchestrator to have a clear app state (important for the Espresso Tests). When I run those Tests from Android Studio everything works fine. But when I try to use the command line I receive errors I can't really understand.

when I try:

./gradlew connectedAndroidTest or connectedDebugAndroidTest

I receive:

Instrumentation run failed due to 'java.lang.IllegalStateException'
com.android.builder.testing.ConnectedDevice > No tests found.[SM-J106H - 
6.0.1] FAILED 
No tests found. This usually means that your test classes are not in the 
form that your test runner expects (e.g. don't inherit from TestCase or lack 
@Test annotations).

and of course all my tests are annotated with @Test.

when I try

adb shell am instrument -w my.package/android.test.InstrumentationTestRunner

I receive

INSTRUMENTATION_STATUS: Error=Unable to find instrumentation info for: 
ComponentInfo{mypackage/myCustomRunner}
INSTRUMENTATION_STATUS_CODE: -1

I use a CustomTestRunner but the Error stays the same.

and when I try to

 adb shell 'CLASSPATH=$(pm path android.support.test.services) app_process / 
 \
 android.support.test.services.shellexecutor.ShellMain am instrument -w -e \
 targetInstrumentation 
 mypackage/myTestRunner \
 android.support.test.orchestrator/.AndroidTestOrchestrator'

then the output is equal to:

Time: 0

OK (0 tests)

Can someone explain to me what I am doing wrong? I can't really understand why nothing works with the command line but inside Android Studio everything is running fine.

/edit

My CustomRunner:

public final class CustomTestRunner extends AndroidJUnitRunner {

private static final String TAG = "CustomTestRunner";

@Override
public void onStart() {

    try {
        TestListener.getInstance().testRunStarted();
    } catch (Exception e) {
        e.printStackTrace();
    }

    runOnMainSync(new Runnable() {
        @Override
        public void run() {
            Context app = CustomTestRunner.this.getTargetContext().getApplicationContext();

            CustomTestRunner.this.disableAnimations(app);
        }
    });

    ActivityLifecycleMonitorRegistry.getInstance().addLifecycleCallback(new ActivityLifecycleCallback() {
        @Override public void onActivityLifecycleChanged(Activity activity, Stage stage) {
            if (stage == Stage.PRE_ON_CREATE) {
                activity.getWindow().addFlags(FLAG_DISMISS_KEYGUARD | FLAG_TURN_SCREEN_ON | FLAG_KEEP_SCREEN_ON);
            }
        }
    });

    RxJavaPlugins.setIoSchedulerHandler(new Function<Scheduler, Scheduler>() {
        @Override
        public Scheduler apply(Scheduler scheduler) throws Exception {
            return Schedulers.from(AsyncTask.THREAD_POOL_EXECUTOR);
        }
    });

    RxJavaPlugins.setComputationSchedulerHandler(new Function<Scheduler, Scheduler>() {
        @Override
        public Scheduler apply(Scheduler scheduler) throws Exception {
            return Schedulers.from(AsyncTask.THREAD_POOL_EXECUTOR);
        }
    });

    RxJavaPlugins.setNewThreadSchedulerHandler(new Function<Scheduler, Scheduler>() {
        @Override
        public Scheduler apply(Scheduler scheduler) throws Exception {
            return Schedulers.from(AsyncTask.THREAD_POOL_EXECUTOR);
        }
    });

    super.onStart();
}


@Override
public void finish(int resultCode, Bundle results) {
    try {
        TestListener.getInstance().testRunFinished();
    } catch (Exception e) {
        e.printStackTrace();
    }

    super.finish(resultCode, results);
    enableAnimations(getContext());
}

private void disableAnimations(Context context) {
    int permStatus = context.checkCallingOrSelfPermission(Manifest.permission.SET_ANIMATION_SCALE);
    if (permStatus == PackageManager.PERMISSION_GRANTED) {
        setSystemAnimationsScale(0.0f);
    }
}

private void enableAnimations(Context context) {
    int permStatus = context.checkCallingOrSelfPermission(Manifest.permission.SET_ANIMATION_SCALE);
    if (permStatus == PackageManager.PERMISSION_GRANTED) {
        setSystemAnimationsScale(1.0f);
    }
}

private void setSystemAnimationsScale(float animationScale) {
    try {
        Class<?> windowManagerStubClazz = Class.forName("android.view.IWindowManager$Stub");
        Method asInterface = windowManagerStubClazz.getDeclaredMethod("asInterface", IBinder.class);
        Class<?> serviceManagerClazz = Class.forName("android.os.ServiceManager");
        Method getService = serviceManagerClazz.getDeclaredMethod("getService", String.class);
        Class<?> windowManagerClazz = Class.forName("android.view.IWindowManager");
        Method setAnimationScales = windowManagerClazz.getDeclaredMethod("setAnimationScales", float[].class);
        Method getAnimationScales = windowManagerClazz.getDeclaredMethod("getAnimationScales");

        IBinder windowManagerBinder = (IBinder) getService.invoke(null, "window");
        Object windowManagerObj = asInterface.invoke(null, windowManagerBinder);
        float[] currentScales = (float[]) getAnimationScales.invoke(windowManagerObj);
        for (int i = 0; i < currentScales.length; i++) {
            currentScales[i] = animationScale;
        }
        setAnimationScales.invoke(windowManagerObj, new Object[]{currentScales});
        Log.d(TAG, "Changed permissions of animations");
    } catch (Exception e) {
        Log.e(TAG, "Could not change animation scale to " + animationScale + " :'(");
    }
}
}

and thats one of my Espresso Test Classes (the DetailView of an item of a visible RecyclerView List)

@RunWith(AndroidJUnit4.class)
public class DetailActivityTest {

private IdlingResource mInitialInformationIdlingResource;

@Before
public void setUp() throws UiObjectNotFoundException, InterruptedException {
    SetupHelper.setUp();

    File tempRealmFile = new File(InstrumentationRegistry.getTargetContext().getFilesDir(), PRODUCT_REALM_DB_FILE_NAME);

    if(tempRealmFile.length() <= 8192 && CustomAssertion.doesViewExist(R.id.countries)) {

        onView(withId(R.id.countries))
                .check(matches(isDisplayed()));
        onData(anything()).inAdapterView(withId(R.id.countries)).atPosition(3).perform(click());

        mInitialInformationIdlingResource = new InitialInformationIdlingResource();
        IdlingRegistry.getInstance().register(mInitialInformationIdlingResource);

        Espresso.onView(withText("OK"))
                .check(matches(isDisplayed()))
                .perform(click());
    }
}

@Test
public void ensureDetailViewWorks() throws UiObjectNotFoundException {
    SetupHelper.checkForDialogs();

    onView(withId(R.id.show_filter_results)).perform(scrollTo());

    onView(withId(R.id.show_filter_results))
            .check(matches(isDisplayed())).perform(scrollTo(), click());

    onView(withId(R.id.resultList)).perform(RecyclerViewActions.actionOnItemAtPosition(1, click()));
    onView(withId(R.id.main_container)).check(matches(isDisplayed()));
    onView(withId(R.id.detail_item_icon)).check(matches(isDisplayed()));
}

}

my build types inside build.gradle

buildTypes {

    debug {
        debuggable true
        minifyEnabled false
        versionNameSuffix "-debug"
        manifestPlaceholders = [HOCKEYAPP_APP_ID: ""]
        testCoverageEnabled true
    }

    release {
        minifyEnabled false
        proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
        signingConfig signingConfigs.release
        versionNameSuffix "-release"
        manifestPlaceholders = [HOCKEYAPP_APP_ID: ""]
    }
}

Answer 1

Look at the logs on your device/emulator. Something in the app/test code is crashing before the testing even starts. "No tests found. This usually means that your test classes are not in the form that your test runner expects." is totally not helpful to you =)